/* * Copyright (c) 2021, Tim Flynn * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include TEST_CASE(decode_ascii) { auto string = AK::utf8_to_utf16("Hello World!11"sv); Utf16View view { string }; size_t valid_code_units = 0; EXPECT(view.validate(valid_code_units)); EXPECT_EQ(valid_code_units, view.length_in_code_units()); auto expected = Array { (u32)72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 49, 49 }; EXPECT_EQ(expected.size(), view.length_in_code_points()); size_t i = 0; for (u32 code_point : view) { EXPECT_EQ(code_point, expected[i++]); } EXPECT_EQ(i, expected.size()); } TEST_CASE(decode_utf8) { auto string = AK::utf8_to_utf16("Привет, мир! 😀 γειά σου κόσμος こんにちは世界"sv); Utf16View view { string }; size_t valid_code_units = 0; EXPECT(view.validate(valid_code_units)); EXPECT_EQ(valid_code_units, view.length_in_code_units()); auto expected = Array { (u32)1055, 1088, 1080, 1074, 1077, 1090, 44, 32, 1084, 1080, 1088, 33, 32, 128512, 32, 947, 949, 953, 940, 32, 963, 959, 965, 32, 954, 972, 963, 956, 959, 962, 32, 12371, 12435, 12395, 12385, 12399, 19990, 30028 }; EXPECT_EQ(expected.size(), view.length_in_code_points()); size_t i = 0; for (u32 code_point : view) { EXPECT_EQ(code_point, expected[i++]); } EXPECT_EQ(i, expected.size()); } TEST_CASE(encode_utf8) { { DeprecatedString utf8_string("Привет, мир! 😀 γειά σου κόσμος こんにちは世界"); auto string = AK::utf8_to_utf16(utf8_string); Utf16View view { string }; EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes), utf8_string); EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No), utf8_string); } { auto encoded = Array { (u16)0xd83d }; Utf16View view { encoded }; EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes), "\xed\xa0\xbd"sv); EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No), "\ufffd"sv); } } TEST_CASE(decode_utf16) { // Same string as the decode_utf8 test. auto encoded = Array { (u16)0x041f, 0x0440, 0x0438, 0x0432, 0x0435, 0x0442, 0x002c, 0x0020, 0x043c, 0x0438, 0x0440, 0x0021, 0x0020, 0xd83d, 0xde00, 0x0020, 0x03b3, 0x03b5, 0x03b9, 0x03ac, 0x0020, 0x03c3, 0x03bf, 0x03c5, 0x0020, 0x03ba, 0x03cc, 0x03c3, 0x03bc, 0x03bf, 0x03c2, 0x0020, 0x3053, 0x3093, 0x306b, 0x3061, 0x306f, 0x4e16, 0x754c }; Utf16View view { encoded }; EXPECT_EQ(encoded.size(), view.length_in_code_units()); size_t valid_code_units = 0; EXPECT(view.validate(valid_code_units)); EXPECT_EQ(valid_code_units, view.length_in_code_units()); auto expected = Array { (u32)1055, 1088, 1080, 1074, 1077, 1090, 44, 32, 1084, 1080, 1088, 33, 32, 128512, 32, 947, 949, 953, 940, 32, 963, 959, 965, 32, 954, 972, 963, 956, 959, 962, 32, 12371, 12435, 12395, 12385, 12399, 19990, 30028 }; EXPECT_EQ(expected.size(), view.length_in_code_points()); size_t i = 0; for (u32 code_point : view) { EXPECT_EQ(code_point, expected[i++]); } EXPECT_EQ(i, expected.size()); } TEST_CASE(iterate_utf16) { auto string = AK::utf8_to_utf16("Привет 😀"sv); Utf16View view { string }; auto iterator = view.begin(); EXPECT(*iterator == 1055); EXPECT(iterator.length_in_code_units() == 1); EXPECT(++iterator != view.end()); EXPECT(*iterator == 1088); EXPECT(iterator.length_in_code_units() == 1); EXPECT(++iterator != view.end()); EXPECT(*iterator == 1080); EXPECT(iterator.length_in_code_units() == 1); EXPECT(++iterator != view.end()); EXPECT(*iterator == 1074); EXPECT(iterator.length_in_code_units() == 1); EXPECT(++iterator != view.end()); EXPECT(*iterator == 1077); EXPECT(iterator.length_in_code_units() == 1); EXPECT(++iterator != view.end()); EXPECT(*iterator == 1090); EXPECT(iterator.length_in_code_units() == 1); EXPECT(++iterator != view.end()); EXPECT(*iterator == 32); EXPECT(iterator.length_in_code_units() == 1); EXPECT(++iterator != view.end()); EXPECT(*iterator == 128512); EXPECT(iterator.length_in_code_units() == 2); EXPECT(++iterator == view.end()); EXPECT_CRASH("Dereferencing Utf16CodePointIterator which is at its end.", [&iterator] { *iterator; return Test::Crash::Failure::DidNotCrash; }); EXPECT_CRASH("Incrementing Utf16CodePointIterator which is at its end.", [&iterator] { ++iterator; return Test::Crash::Failure::DidNotCrash; }); } TEST_CASE(validate_invalid_utf16) { size_t valid_code_units = 0; { // Lonely high surrogate. auto invalid = Array { (u16)0xd800 }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); invalid = Array { (u16)0xdbff }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); } { // Lonely low surrogate. auto invalid = Array { (u16)0xdc00 }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); invalid = Array { (u16)0xdfff }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); } { // High surrogate followed by non-surrogate. auto invalid = Array { (u16)0xd800, 0 }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); invalid = Array { (u16)0xd800, 0xe000 }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); } { // High surrogate followed by high surrogate. auto invalid = Array { (u16)0xd800, 0xd800 }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); invalid = Array { (u16)0xd800, 0xdbff }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 0); } { // Valid UTF-16 followed by invalid code units. auto invalid = Array { (u16)0x41, 0x41, 0xd800 }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 2); invalid = Array { (u16)0x41, 0x41, 0xd800 }; EXPECT(!Utf16View(invalid).validate(valid_code_units)); EXPECT(valid_code_units == 2); } } TEST_CASE(decode_invalid_utf16) { { // Lonely high surrogate. auto invalid = Array { (u16)0x41, 0x42, 0xd800 }; Utf16View view { invalid }; EXPECT_EQ(invalid.size(), view.length_in_code_units()); auto expected = Array { (u32)0x41, 0x42, 0xfffd }; EXPECT_EQ(expected.size(), view.length_in_code_points()); size_t i = 0; for (u32 code_point : view) { EXPECT_EQ(code_point, expected[i++]); } EXPECT_EQ(i, expected.size()); } { // Lonely low surrogate. auto invalid = Array { (u16)0x41, 0x42, 0xdc00 }; Utf16View view { invalid }; EXPECT_EQ(invalid.size(), view.length_in_code_units()); auto expected = Array { (u32)0x41, 0x42, 0xfffd }; EXPECT_EQ(expected.size(), view.length_in_code_points()); size_t i = 0; for (u32 code_point : view) { EXPECT_EQ(code_point, expected[i++]); } EXPECT_EQ(i, expected.size()); } { // High surrogate followed by non-surrogate. auto invalid = Array { (u16)0x41, 0x42, 0xd800, 0 }; Utf16View view { invalid }; EXPECT_EQ(invalid.size(), view.length_in_code_units()); auto expected = Array { (u32)0x41, 0x42, 0xfffd, 0 }; EXPECT_EQ(expected.size(), view.length_in_code_points()); size_t i = 0; for (u32 code_point : view) { EXPECT_EQ(code_point, expected[i++]); } EXPECT_EQ(i, expected.size()); } { // High surrogate followed by high surrogate. auto invalid = Array { (u16)0x41, 0x42, 0xd800, 0xd800 }; Utf16View view { invalid }; EXPECT_EQ(invalid.size(), view.length_in_code_units()); auto expected = Array { (u32)0x41, 0x42, 0xfffd, 0xfffd }; EXPECT_EQ(expected.size(), view.length_in_code_points()); size_t i = 0; for (u32 code_point : view) { EXPECT_EQ(code_point, expected[i++]); } EXPECT_EQ(i, expected.size()); } } TEST_CASE(substring_view) { auto string = AK::utf8_to_utf16("Привет 😀"sv); { Utf16View view { string }; view = view.substring_view(7, 2); EXPECT(view.length_in_code_units() == 2); EXPECT_EQ(view.to_utf8(), "😀"sv); } { Utf16View view { string }; view = view.substring_view(7, 1); EXPECT(view.length_in_code_units() == 1); EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes), "\xed\xa0\xbd"sv); EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No), "\ufffd"sv); } }