tokenizer.js

function shouldMatchTokens(result, tokens) {
  for (var index = 0; index < result.length; index++) {
    equals(result[index].name, tokens[index]);
  }
}
function shouldBeToken(result, name, text) {
  equals(result.name, name);
  equals(result.text, text);
}

describe('Tokenizer', function () {
  if (!Handlebars.Parser) {
    return;
  }

  function tokenize(template) {
    var parser = Handlebars.Parser,
      lexer = parser.lexer;

    lexer.setInput(template);
    var out = [],
      token;

    while ((token = lexer.lex())) {
      var result = parser.terminals_[token] || token;
      if (!result || result === 'EOF' || result === 'INVALID') {
        break;
      }
      out.push({ name: result, text: lexer.yytext });
    }

    return out;
  }

  it('tokenizes a simple mustache as "OPEN ID CLOSE"', function () {
    var result = tokenize('{{foo}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'CLOSE']);
    shouldBeToken(result[1], 'ID', 'foo');
  });

  it('supports unescaping with &', function () {
    var result = tokenize('{{&bar}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'CLOSE']);

    shouldBeToken(result[0], 'OPEN', '{{&');
    shouldBeToken(result[1], 'ID', 'bar');
  });

  it('supports unescaping with {{{', function () {
    var result = tokenize('{{{bar}}}');
    shouldMatchTokens(result, ['OPEN_UNESCAPED', 'ID', 'CLOSE_UNESCAPED']);

    shouldBeToken(result[1], 'ID', 'bar');
  });

  it('supports escaping delimiters', function () {
    var result = tokenize('{{foo}} \\{{bar}} {{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);

    shouldBeToken(result[3], 'CONTENT', ' ');
    shouldBeToken(result[4], 'CONTENT', '{{bar}} ');
  });

  it('supports escaping multiple delimiters', function () {
    var result = tokenize('{{foo}} \\{{bar}} \\{{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'CONTENT',
      'CONTENT',
    ]);

    shouldBeToken(result[3], 'CONTENT', ' ');
    shouldBeToken(result[4], 'CONTENT', '{{bar}} ');
    shouldBeToken(result[5], 'CONTENT', '{{baz}}');
  });

  it('supports escaping a triple stash', function () {
    var result = tokenize('{{foo}} \\{{{bar}}} {{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);

    shouldBeToken(result[4], 'CONTENT', '{{{bar}}} ');
  });

  it('supports escaping escape character', function () {
    var result = tokenize('{{foo}} \\\\{{bar}} {{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);

    shouldBeToken(result[3], 'CONTENT', ' \\');
    shouldBeToken(result[5], 'ID', 'bar');
  });

  it('supports escaping multiple escape characters', function () {
    var result = tokenize('{{foo}} \\\\{{bar}} \\\\{{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);

    shouldBeToken(result[3], 'CONTENT', ' \\');
    shouldBeToken(result[5], 'ID', 'bar');
    shouldBeToken(result[7], 'CONTENT', ' \\');
    shouldBeToken(result[9], 'ID', 'baz');
  });

  it('supports escaped mustaches after escaped escape characters', function () {
    var result = tokenize('{{foo}} \\\\{{bar}} \\{{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'CONTENT',
      'CONTENT',
    ]);

    shouldBeToken(result[3], 'CONTENT', ' \\');
    shouldBeToken(result[4], 'OPEN', '{{');
    shouldBeToken(result[5], 'ID', 'bar');
    shouldBeToken(result[7], 'CONTENT', ' ');
    shouldBeToken(result[8], 'CONTENT', '{{baz}}');
  });

  it('supports escaped escape characters after escaped mustaches', function () {
    var result = tokenize('{{foo}} \\{{bar}} \\\\{{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'CONTENT',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);

    shouldBeToken(result[4], 'CONTENT', '{{bar}} ');
    shouldBeToken(result[5], 'CONTENT', '\\');
    shouldBeToken(result[6], 'OPEN', '{{');
    shouldBeToken(result[7], 'ID', 'baz');
  });

  it('supports escaped escape character on a triple stash', function () {
    var result = tokenize('{{foo}} \\\\{{{bar}}} {{baz}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN_UNESCAPED',
      'ID',
      'CLOSE_UNESCAPED',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);

    shouldBeToken(result[3], 'CONTENT', ' \\');
    shouldBeToken(result[5], 'ID', 'bar');
  });

  it('tokenizes a simple path', function () {
    var result = tokenize('{{foo/bar}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'SEP', 'ID', 'CLOSE']);
  });

  it('allows dot notation', function () {
    var result = tokenize('{{foo.bar}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'SEP', 'ID', 'CLOSE']);

    shouldMatchTokens(tokenize('{{foo.bar.baz}}'), [
      'OPEN',
      'ID',
      'SEP',
      'ID',
      'SEP',
      'ID',
      'CLOSE',
    ]);
  });

  it('allows path literals with []', function () {
    var result = tokenize('{{foo.[bar]}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'SEP', 'ID', 'CLOSE']);
  });

  it('allows multiple path literals on a line with []', function () {
    var result = tokenize('{{foo.[bar]}}{{foo.[baz]}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'SEP',
      'ID',
      'CLOSE',
      'OPEN',
      'ID',
      'SEP',
      'ID',
      'CLOSE',
    ]);
  });

  it('allows escaped literals in []', function () {
    var result = tokenize('{{foo.[bar\\]]}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'SEP', 'ID', 'CLOSE']);
  });

  it('tokenizes {{.}} as OPEN ID CLOSE', function () {
    var result = tokenize('{{.}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'CLOSE']);
  });

  it('tokenizes a path as "OPEN (ID SEP)* ID CLOSE"', function () {
    var result = tokenize('{{../foo/bar}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'SEP',
      'ID',
      'SEP',
      'ID',
      'CLOSE',
    ]);
    shouldBeToken(result[1], 'ID', '..');
  });

  it('tokenizes a path with .. as a parent path', function () {
    var result = tokenize('{{../foo.bar}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'SEP',
      'ID',
      'SEP',
      'ID',
      'CLOSE',
    ]);
    shouldBeToken(result[1], 'ID', '..');
  });

  it('tokenizes a path with this/foo as OPEN ID SEP ID CLOSE', function () {
    var result = tokenize('{{this/foo}}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'SEP', 'ID', 'CLOSE']);
    shouldBeToken(result[1], 'ID', 'this');
    shouldBeToken(result[3], 'ID', 'foo');
  });

  it('tokenizes a simple mustache with spaces as "OPEN ID CLOSE"', function () {
    var result = tokenize('{{  foo  }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'CLOSE']);
    shouldBeToken(result[1], 'ID', 'foo');
  });

  it('tokenizes a simple mustache with line breaks as "OPEN ID ID CLOSE"', function () {
    var result = tokenize('{{  foo  \n   bar }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'ID', 'CLOSE']);
    shouldBeToken(result[1], 'ID', 'foo');
  });

  it('tokenizes raw content as "CONTENT"', function () {
    var result = tokenize('foo {{ bar }} baz');
    shouldMatchTokens(result, ['CONTENT', 'OPEN', 'ID', 'CLOSE', 'CONTENT']);
    shouldBeToken(result[0], 'CONTENT', 'foo ');
    shouldBeToken(result[4], 'CONTENT', ' baz');
  });

  it('tokenizes a partial as "OPEN_PARTIAL ID CLOSE"', function () {
    var result = tokenize('{{> foo}}');
    shouldMatchTokens(result, ['OPEN_PARTIAL', 'ID', 'CLOSE']);
  });

  it('tokenizes a partial with context as "OPEN_PARTIAL ID ID CLOSE"', function () {
    var result = tokenize('{{> foo bar }}');
    shouldMatchTokens(result, ['OPEN_PARTIAL', 'ID', 'ID', 'CLOSE']);
  });

  it('tokenizes a partial without spaces as "OPEN_PARTIAL ID CLOSE"', function () {
    var result = tokenize('{{>foo}}');
    shouldMatchTokens(result, ['OPEN_PARTIAL', 'ID', 'CLOSE']);
  });

  it('tokenizes a partial space at the }); as "OPEN_PARTIAL ID CLOSE"', function () {
    var result = tokenize('{{>foo  }}');
    shouldMatchTokens(result, ['OPEN_PARTIAL', 'ID', 'CLOSE']);
  });

  it('tokenizes a partial space at the }); as "OPEN_PARTIAL ID CLOSE"', function () {
    var result = tokenize('{{>foo/bar.baz  }}');
    shouldMatchTokens(result, [
      'OPEN_PARTIAL',
      'ID',
      'SEP',
      'ID',
      'SEP',
      'ID',
      'CLOSE',
    ]);
  });

  it('tokenizes partial block declarations', function () {
    var result = tokenize('{{#> foo}}');
    shouldMatchTokens(result, ['OPEN_PARTIAL_BLOCK', 'ID', 'CLOSE']);
  });
  it('tokenizes a comment as "COMMENT"', function () {
    var result = tokenize('foo {{! this is a comment }} bar {{ baz }}');
    shouldMatchTokens(result, [
      'CONTENT',
      'COMMENT',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);
    shouldBeToken(result[1], 'COMMENT', '{{! this is a comment }}');
  });

  it('tokenizes a block comment as "COMMENT"', function () {
    var result = tokenize('foo {{!-- this is a {{comment}} --}} bar {{ baz }}');
    shouldMatchTokens(result, [
      'CONTENT',
      'COMMENT',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);
    shouldBeToken(result[1], 'COMMENT', '{{!-- this is a {{comment}} --}}');
  });

  it('tokenizes a block comment with whitespace as "COMMENT"', function () {
    var result = tokenize(
      'foo {{!-- this is a\n{{comment}}\n--}} bar {{ baz }}'
    );
    shouldMatchTokens(result, [
      'CONTENT',
      'COMMENT',
      'CONTENT',
      'OPEN',
      'ID',
      'CLOSE',
    ]);
    shouldBeToken(result[1], 'COMMENT', '{{!-- this is a\n{{comment}}\n--}}');
  });

  it('tokenizes open and closing blocks as OPEN_BLOCK, ID, CLOSE ..., OPEN_ENDBLOCK ID CLOSE', function () {
    var result = tokenize('{{#foo}}content{{/foo}}');
    shouldMatchTokens(result, [
      'OPEN_BLOCK',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN_ENDBLOCK',
      'ID',
      'CLOSE',
    ]);
  });

  it('tokenizes directives', function () {
    shouldMatchTokens(tokenize('{{#*foo}}content{{/foo}}'), [
      'OPEN_BLOCK',
      'ID',
      'CLOSE',
      'CONTENT',
      'OPEN_ENDBLOCK',
      'ID',
      'CLOSE',
    ]);
    shouldMatchTokens(tokenize('{{*foo}}'), ['OPEN', 'ID', 'CLOSE']);
  });

  it('tokenizes inverse sections as "INVERSE"', function () {
    shouldMatchTokens(tokenize('{{^}}'), ['INVERSE']);
    shouldMatchTokens(tokenize('{{else}}'), ['INVERSE']);
    shouldMatchTokens(tokenize('{{ else }}'), ['INVERSE']);
  });

  it('tokenizes inverse sections with ID as "OPEN_INVERSE ID CLOSE"', function () {
    var result = tokenize('{{^foo}}');
    shouldMatchTokens(result, ['OPEN_INVERSE', 'ID', 'CLOSE']);
    shouldBeToken(result[1], 'ID', 'foo');
  });

  it('tokenizes inverse sections with ID and spaces as "OPEN_INVERSE ID CLOSE"', function () {
    var result = tokenize('{{^ foo  }}');
    shouldMatchTokens(result, ['OPEN_INVERSE', 'ID', 'CLOSE']);
    shouldBeToken(result[1], 'ID', 'foo');
  });

  it('tokenizes mustaches with params as "OPEN ID ID ID CLOSE"', function () {
    var result = tokenize('{{ foo bar baz }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'ID', 'ID', 'CLOSE']);
    shouldBeToken(result[1], 'ID', 'foo');
    shouldBeToken(result[2], 'ID', 'bar');
    shouldBeToken(result[3], 'ID', 'baz');
  });

  it('tokenizes mustaches with String params as "OPEN ID ID STRING CLOSE"', function () {
    var result = tokenize('{{ foo bar "baz" }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'ID', 'STRING', 'CLOSE']);
    shouldBeToken(result[3], 'STRING', 'baz');
  });

  it('tokenizes mustaches with String params using single quotes as "OPEN ID ID STRING CLOSE"', function () {
    var result = tokenize("{{ foo bar 'baz' }}");
    shouldMatchTokens(result, ['OPEN', 'ID', 'ID', 'STRING', 'CLOSE']);
    shouldBeToken(result[3], 'STRING', 'baz');
  });

  it('tokenizes String params with spaces inside as "STRING"', function () {
    var result = tokenize('{{ foo bar "baz bat" }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'ID', 'STRING', 'CLOSE']);
    shouldBeToken(result[3], 'STRING', 'baz bat');
  });

  it('tokenizes String params with escapes quotes as STRING', function () {
    var result = tokenize('{{ foo "bar\\"baz" }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'STRING', 'CLOSE']);
    shouldBeToken(result[2], 'STRING', 'bar"baz');
  });

  it('tokenizes String params using single quotes with escapes quotes as STRING', function () {
    var result = tokenize("{{ foo 'bar\\'baz' }}");
    shouldMatchTokens(result, ['OPEN', 'ID', 'STRING', 'CLOSE']);
    shouldBeToken(result[2], 'STRING', "bar'baz");
  });

  it('tokenizes numbers', function () {
    var result = tokenize('{{ foo 1 }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'NUMBER', 'CLOSE']);
    shouldBeToken(result[2], 'NUMBER', '1');

    result = tokenize('{{ foo 1.1 }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'NUMBER', 'CLOSE']);
    shouldBeToken(result[2], 'NUMBER', '1.1');

    result = tokenize('{{ foo -1 }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'NUMBER', 'CLOSE']);
    shouldBeToken(result[2], 'NUMBER', '-1');

    result = tokenize('{{ foo -1.1 }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'NUMBER', 'CLOSE']);
    shouldBeToken(result[2], 'NUMBER', '-1.1');
  });

  it('tokenizes booleans', function () {
    var result = tokenize('{{ foo true }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'BOOLEAN', 'CLOSE']);
    shouldBeToken(result[2], 'BOOLEAN', 'true');

    result = tokenize('{{ foo false }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'BOOLEAN', 'CLOSE']);
    shouldBeToken(result[2], 'BOOLEAN', 'false');
  });

  it('tokenizes undefined and null', function () {
    var result = tokenize('{{ foo undefined null }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'UNDEFINED', 'NULL', 'CLOSE']);
    shouldBeToken(result[2], 'UNDEFINED', 'undefined');
    shouldBeToken(result[3], 'NULL', 'null');
  });

  it('tokenizes hash arguments', function () {
    var result = tokenize('{{ foo bar=baz }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'ID', 'EQUALS', 'ID', 'CLOSE']);

    result = tokenize('{{ foo bar baz=bat }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'ID',
      'CLOSE',
    ]);

    result = tokenize('{{ foo bar baz=1 }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'NUMBER',
      'CLOSE',
    ]);

    result = tokenize('{{ foo bar baz=true }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'BOOLEAN',
      'CLOSE',
    ]);

    result = tokenize('{{ foo bar baz=false }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'BOOLEAN',
      'CLOSE',
    ]);

    result = tokenize('{{ foo bar\n  baz=bat }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'ID',
      'CLOSE',
    ]);

    result = tokenize('{{ foo bar baz="bat" }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'STRING',
      'CLOSE',
    ]);

    result = tokenize('{{ foo bar baz="bat" bam=wot }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'STRING',
      'ID',
      'EQUALS',
      'ID',
      'CLOSE',
    ]);

    result = tokenize('{{foo omg bar=baz bat="bam"}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'ID',
      'EQUALS',
      'ID',
      'ID',
      'EQUALS',
      'STRING',
      'CLOSE',
    ]);
    shouldBeToken(result[2], 'ID', 'omg');
  });

  it('tokenizes special @ identifiers', function () {
    var result = tokenize('{{ @foo }}');
    shouldMatchTokens(result, ['OPEN', 'DATA', 'ID', 'CLOSE']);
    shouldBeToken(result[2], 'ID', 'foo');

    result = tokenize('{{ foo @bar }}');
    shouldMatchTokens(result, ['OPEN', 'ID', 'DATA', 'ID', 'CLOSE']);
    shouldBeToken(result[3], 'ID', 'bar');

    result = tokenize('{{ foo bar=@baz }}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'ID',
      'EQUALS',
      'DATA',
      'ID',
      'CLOSE',
    ]);
    shouldBeToken(result[5], 'ID', 'baz');
  });

  it('does not time out in a mustache with a single } followed by EOF', function () {
    shouldMatchTokens(tokenize('{{foo}'), ['OPEN', 'ID']);
  });

  it('does not time out in a mustache when invalid ID characters are used', function () {
    shouldMatchTokens(tokenize('{{foo & }}'), ['OPEN', 'ID']);
  });

  it('tokenizes subexpressions', function () {
    var result = tokenize('{{foo (bar)}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'OPEN_SEXPR',
      'ID',
      'CLOSE_SEXPR',
      'CLOSE',
    ]);
    shouldBeToken(result[1], 'ID', 'foo');
    shouldBeToken(result[3], 'ID', 'bar');

    result = tokenize('{{foo (a-x b-y)}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'OPEN_SEXPR',
      'ID',
      'ID',
      'CLOSE_SEXPR',
      'CLOSE',
    ]);
    shouldBeToken(result[1], 'ID', 'foo');
    shouldBeToken(result[3], 'ID', 'a-x');
    shouldBeToken(result[4], 'ID', 'b-y');
  });

  it('tokenizes nested subexpressions', function () {
    var result = tokenize('{{foo (bar (lol rofl)) (baz)}}');
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'OPEN_SEXPR',
      'ID',
      'OPEN_SEXPR',
      'ID',
      'ID',
      'CLOSE_SEXPR',
      'CLOSE_SEXPR',
      'OPEN_SEXPR',
      'ID',
      'CLOSE_SEXPR',
      'CLOSE',
    ]);
    shouldBeToken(result[3], 'ID', 'bar');
    shouldBeToken(result[5], 'ID', 'lol');
    shouldBeToken(result[6], 'ID', 'rofl');
    shouldBeToken(result[10], 'ID', 'baz');
  });

  it('tokenizes nested subexpressions: literals', function () {
    var result = tokenize(
      '{{foo (bar (lol true) false) (baz 1) (blah \'b\') (blorg "c")}}'
    );
    shouldMatchTokens(result, [
      'OPEN',
      'ID',
      'OPEN_SEXPR',
      'ID',
      'OPEN_SEXPR',
      'ID',
      'BOOLEAN',
      'CLOSE_SEXPR',
      'BOOLEAN',
      'CLOSE_SEXPR',
      'OPEN_SEXPR',
      'ID',
      'NUMBER',
      'CLOSE_SEXPR',
      'OPEN_SEXPR',
      'ID',
      'STRING',
      'CLOSE_SEXPR',
      'OPEN_SEXPR',
      'ID',
      'STRING',
      'CLOSE_SEXPR',
      'CLOSE',
    ]);
  });

  it('tokenizes block params', function () {
    var result = tokenize('{{#foo as |bar|}}');
    shouldMatchTokens(result, [
      'OPEN_BLOCK',
      'ID',
      'OPEN_BLOCK_PARAMS',
      'ID',
      'CLOSE_BLOCK_PARAMS',
      'CLOSE',
    ]);

    result = tokenize('{{#foo as |bar baz|}}');
    shouldMatchTokens(result, [
      'OPEN_BLOCK',
      'ID',
      'OPEN_BLOCK_PARAMS',
      'ID',
      'ID',
      'CLOSE_BLOCK_PARAMS',
      'CLOSE',
    ]);

    result = tokenize('{{#foo as | bar baz |}}');
    shouldMatchTokens(result, [
      'OPEN_BLOCK',
      'ID',
      'OPEN_BLOCK_PARAMS',
      'ID',
      'ID',
      'CLOSE_BLOCK_PARAMS',
      'CLOSE',
    ]);

    result = tokenize('{{#foo as as | bar baz |}}');
    shouldMatchTokens(result, [
      'OPEN_BLOCK',
      'ID',
      'ID',
      'OPEN_BLOCK_PARAMS',
      'ID',
      'ID',
      'CLOSE_BLOCK_PARAMS',
      'CLOSE',
    ]);

    result = tokenize('{{else foo as |bar baz|}}');
    shouldMatchTokens(result, [
      'OPEN_INVERSE_CHAIN',
      'ID',
      'OPEN_BLOCK_PARAMS',
      'ID',
      'ID',
      'CLOSE_BLOCK_PARAMS',
      'CLOSE',
    ]);
  });

  it('tokenizes raw blocks', function () {
    var result = tokenize(
      '{{{{a}}}} abc {{{{/a}}}} aaa {{{{a}}}} abc {{{{/a}}}}'
    );
    shouldMatchTokens(result, [
      'OPEN_RAW_BLOCK',
      'ID',
      'CLOSE_RAW_BLOCK',
      'CONTENT',
      'END_RAW_BLOCK',
      'CONTENT',
      'OPEN_RAW_BLOCK',
      'ID',
      'CLOSE_RAW_BLOCK',
      'CONTENT',
      'END_RAW_BLOCK',
    ]);
  });
});