/* * Copyright (c) 2021, David Tuin * * SPDX-License-Identifier: BSD-2-Clause */ #include #include TEST_CASE(invalid_unicode_only) { char const* code = "\xEA\xFD"; auto lexer = JS::Lexer(code); auto token = lexer.next(); EXPECT_EQ(token.type(), JS::TokenType::Invalid); // After this we can get as many eof tokens as we like. for (auto i = 0; i < 10; i++) { auto eof_token = lexer.next(); EXPECT_EQ(eof_token.type(), JS::TokenType::Eof); } } TEST_CASE(long_invalid_unicode) { char const* code = "\xF7"; auto lexer = JS::Lexer(code); auto token = lexer.next(); EXPECT_EQ(token.type(), JS::TokenType::Invalid); // After this we can get as many eof tokens as we like. for (auto i = 0; i < 10; i++) { auto eof_token = lexer.next(); EXPECT_EQ(eof_token.type(), JS::TokenType::Eof); } } TEST_CASE(invalid_unicode_and_valid_code) { char const* code = "\xEA\xFDthrow 1;"; auto lexer = JS::Lexer(code); auto invalid_token = lexer.next(); EXPECT_EQ(invalid_token.type(), JS::TokenType::Invalid); // 0xEA is the start of a three character unicode code point thus it consumes the 't'. auto token_after = lexer.next(); EXPECT_EQ(token_after.value(), "hrow"); } TEST_CASE(long_invalid_unicode_and_valid_code) { char const* code = "\xF7throw 1;"; auto lexer = JS::Lexer(code); auto invalid_token = lexer.next(); EXPECT_EQ(invalid_token.type(), JS::TokenType::Invalid); // 0xF7 is the start of a four character unicode code point thus it consumes 'thr'. auto token_after = lexer.next(); EXPECT_EQ(token_after.value(), "ow"); } TEST_CASE(invalid_unicode_after_valid_code_and_before_eof) { char const* code = "let \xEA\xFD;"; auto lexer = JS::Lexer(code); auto let_token = lexer.next(); EXPECT_EQ(let_token.type(), JS::TokenType::Let); auto invalid_token = lexer.next(); EXPECT_EQ(invalid_token.type(), JS::TokenType::Invalid); // It should still get the valid trivia in front. EXPECT_EQ(invalid_token.trivia(), " "); // After this we can get as many eof tokens as we like. for (auto i = 0; i < 10; i++) { auto eof_token = lexer.next(); EXPECT_EQ(eof_token.type(), JS::TokenType::Eof); } }