summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--AK/Forward.h2
-rw-r--r--AK/Utf16View.cpp197
-rw-r--r--AK/Utf16View.h113
-rw-r--r--Tests/AK/CMakeLists.txt1
-rw-r--r--Tests/AK/TestUtf16.cpp282
5 files changed, 595 insertions, 0 deletions
diff --git a/AK/Forward.h b/AK/Forward.h
index 408b517ad7..8d4a9b5e8a 100644
--- a/AK/Forward.h
+++ b/AK/Forward.h
@@ -29,6 +29,7 @@ class StringView;
class Time;
class URL;
class FlyString;
+class Utf16View;
class Utf32View;
class Utf8View;
class InputStream;
@@ -168,6 +169,7 @@ using AK::StringView;
using AK::Time;
using AK::Traits;
using AK::URL;
+using AK::Utf16View;
using AK::Utf32View;
using AK::Utf8View;
using AK::Vector;
diff --git a/AK/Utf16View.cpp b/AK/Utf16View.cpp
new file mode 100644
index 0000000000..83d95d4543
--- /dev/null
+++ b/AK/Utf16View.cpp
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/StringBuilder.h>
+#include <AK/StringView.h>
+#include <AK/Utf16View.h>
+#include <AK/Utf8View.h>
+
+namespace AK {
+
+static constexpr u16 high_surrogate_min = 0xd800;
+static constexpr u16 high_surrogate_max = 0xdbff;
+static constexpr u16 low_surrogate_min = 0xdc00;
+static constexpr u16 low_surrogate_max = 0xdfff;
+static constexpr u32 replacement_code_point = 0xfffd;
+static constexpr u32 first_supplementary_plane_code_point = 0x10000;
+
+Vector<u16> utf8_to_utf16(StringView const& utf8_view)
+{
+ return utf8_to_utf16(Utf8View { utf8_view });
+}
+
+Vector<u16> utf8_to_utf16(Utf8View const& utf8_view)
+{
+ Vector<u16> utf16_data;
+
+ for (auto code_point : utf8_view) {
+ if (code_point < first_supplementary_plane_code_point) {
+ utf16_data.append(static_cast<u16>(code_point));
+ } else {
+ code_point -= first_supplementary_plane_code_point;
+ utf16_data.append(static_cast<u16>(high_surrogate_min | (code_point >> 10)));
+ utf16_data.append(static_cast<u16>(low_surrogate_min | (code_point & 0x3ff)));
+ }
+ }
+
+ return utf16_data;
+}
+
+bool Utf16View::is_high_surrogate(u16 code_unit)
+{
+ return (code_unit >= high_surrogate_min) && (code_unit <= high_surrogate_max);
+}
+
+bool Utf16View::is_low_surrogate(u16 code_unit)
+{
+ return (code_unit >= low_surrogate_min) && (code_unit <= low_surrogate_max);
+}
+
+u32 Utf16View::decode_surrogate_pair(u16 high_surrogate, u16 low_surrogate)
+{
+ VERIFY(is_high_surrogate(high_surrogate));
+ VERIFY(is_low_surrogate(low_surrogate));
+
+ return ((high_surrogate - high_surrogate_min) << 10) + (low_surrogate - low_surrogate_min) + first_supplementary_plane_code_point;
+}
+
+String Utf16View::to_utf8(AllowInvalidCodeUnits allow_invalid_code_units) const
+{
+ StringBuilder builder;
+
+ if (allow_invalid_code_units == AllowInvalidCodeUnits::Yes) {
+ for (auto const* ptr = begin_ptr(); ptr < end_ptr(); ++ptr) {
+ if (is_high_surrogate(*ptr)) {
+ auto const* next = ptr + 1;
+
+ if ((next < end_ptr()) && is_low_surrogate(*next)) {
+ auto code_point = decode_surrogate_pair(*ptr, *next);
+ builder.append_code_point(code_point);
+ ++ptr;
+ continue;
+ }
+ }
+
+ builder.append_code_point(static_cast<u32>(*ptr));
+ }
+ } else {
+ for (auto code_point : *this)
+ builder.append_code_point(code_point);
+ }
+
+ return builder.build();
+}
+
+size_t Utf16View::length_in_code_points() const
+{
+ if (!m_length_in_code_points.has_value())
+ m_length_in_code_points = calculate_length_in_code_points();
+ return *m_length_in_code_points;
+}
+
+u16 Utf16View::code_unit_at(size_t index) const
+{
+ VERIFY(index < length_in_code_units());
+ return m_code_units[index];
+}
+
+Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_length) const
+{
+ VERIFY(!Checked<size_t>::addition_would_overflow(code_unit_offset, code_unit_length));
+ VERIFY(code_unit_offset + code_unit_length <= length_in_code_units());
+
+ return Utf16View { m_code_units.slice(code_unit_offset, code_unit_length) };
+}
+
+bool Utf16View::validate(size_t& valid_code_units) const
+{
+ valid_code_units = 0;
+
+ for (auto const* ptr = begin_ptr(); ptr < end_ptr(); ++ptr) {
+ if (is_high_surrogate(*ptr)) {
+ if ((++ptr >= end_ptr()) || !is_low_surrogate(*ptr))
+ return false;
+ ++valid_code_units;
+ } else if (is_low_surrogate(*ptr)) {
+ return false;
+ }
+
+ ++valid_code_units;
+ }
+
+ return true;
+}
+
+size_t Utf16View::calculate_length_in_code_points() const
+{
+ size_t code_points = 0;
+ for ([[maybe_unused]] auto code_point : *this)
+ ++code_points;
+ return code_points;
+}
+
+bool Utf16View::operator==(Utf16View const& other) const
+{
+ if (length_in_code_units() == 0)
+ return other.length_in_code_units() == 0;
+ if (length_in_code_units() != other.length_in_code_units())
+ return false;
+
+ for (size_t i = 0; i < length_in_code_units(); ++i) {
+ if (m_code_units[i] != other.m_code_units[i])
+ return false;
+ }
+
+ return true;
+}
+
+Utf16CodePointIterator& Utf16CodePointIterator::operator++()
+{
+ size_t code_units = length_in_code_units();
+
+ if (code_units > m_remaining_code_units) {
+ // If there aren't enough code units remaining, skip to the end.
+ m_ptr += m_remaining_code_units;
+ m_remaining_code_units = 0;
+ } else {
+ m_ptr += code_units;
+ m_remaining_code_units -= code_units;
+ }
+
+ return *this;
+}
+
+u32 Utf16CodePointIterator::operator*() const
+{
+ VERIFY(m_remaining_code_units > 0);
+
+ if (Utf16View::is_high_surrogate(*m_ptr)) {
+ if ((m_remaining_code_units > 1) && Utf16View::is_low_surrogate(*(m_ptr + 1)))
+ return Utf16View::decode_surrogate_pair(*m_ptr, *(m_ptr + 1));
+ return replacement_code_point;
+ } else if (Utf16View::is_low_surrogate(*m_ptr)) {
+ return replacement_code_point;
+ }
+
+ return static_cast<u32>(*m_ptr);
+}
+
+size_t Utf16CodePointIterator::length_in_code_units() const
+{
+ VERIFY(m_remaining_code_units > 0);
+
+ if (Utf16View::is_high_surrogate(*m_ptr)) {
+ if ((m_remaining_code_units > 1) && Utf16View::is_low_surrogate(*(m_ptr + 1)))
+ return 2;
+ }
+
+ // If this return is reached, either the encoded code point is a valid single code unit, or that
+ // code point is invalid (e.g. began with a low surrogate, or a low surrogate did not follow a
+ // high surrogate). In the latter case, a single replacement code unit will be used.
+ return 1;
+}
+
+}
diff --git a/AK/Utf16View.h b/AK/Utf16View.h
new file mode 100644
index 0000000000..09f269df0f
--- /dev/null
+++ b/AK/Utf16View.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Forward.h>
+#include <AK/Optional.h>
+#include <AK/Span.h>
+#include <AK/String.h>
+#include <AK/Types.h>
+#include <AK/Vector.h>
+
+namespace AK {
+
+Vector<u16> utf8_to_utf16(StringView const&);
+Vector<u16> utf8_to_utf16(Utf8View const&);
+
+class Utf16View;
+
+class Utf16CodePointIterator {
+ friend class Utf16View;
+
+public:
+ Utf16CodePointIterator() = default;
+ ~Utf16CodePointIterator() = default;
+
+ bool operator==(Utf16CodePointIterator const& other) const
+ {
+ return (m_ptr == other.m_ptr) && (m_remaining_code_units == other.m_remaining_code_units);
+ }
+
+ bool operator!=(Utf16CodePointIterator const& other) const
+ {
+ return !(*this == other);
+ }
+
+ Utf16CodePointIterator& operator++();
+ u32 operator*() const;
+
+ size_t length_in_code_units() const;
+
+private:
+ Utf16CodePointIterator(u16 const* ptr, size_t length)
+ : m_ptr(ptr)
+ , m_remaining_code_units(length)
+ {
+ }
+
+ u16 const* m_ptr { nullptr };
+ size_t m_remaining_code_units { 0 };
+};
+
+class Utf16View {
+public:
+ static bool is_high_surrogate(u16);
+ static bool is_low_surrogate(u16);
+ static u32 decode_surrogate_pair(u16 high_surrogate, u16 low_surrogate);
+
+ Utf16View() = default;
+ ~Utf16View() = default;
+
+ explicit Utf16View(Span<u16 const> code_units)
+ : m_code_units(code_units)
+ {
+ }
+
+ Utf16View& operator=(Utf16View const&) = default;
+ Utf16View& operator=(Utf16View&&) = default;
+ bool operator==(Utf16View const& other) const;
+
+ enum class AllowInvalidCodeUnits {
+ Yes,
+ No,
+ };
+
+ String to_utf8(AllowInvalidCodeUnits = AllowInvalidCodeUnits::No) const;
+
+ bool is_empty() const { return m_code_units.is_empty(); }
+ size_t length_in_code_units() const { return m_code_units.size(); }
+ size_t length_in_code_points() const;
+
+ Utf16CodePointIterator begin() const { return { begin_ptr(), m_code_units.size() }; }
+ Utf16CodePointIterator end() const { return { end_ptr(), 0 }; }
+
+ u16 const* data() const { return m_code_units.data(); }
+ u16 code_unit_at(size_t index) const;
+
+ Utf16View substring_view(size_t code_unit_offset, size_t code_unit_length) const;
+ Utf16View substring_view(size_t code_unit_offset) const { return substring_view(code_unit_offset, length_in_code_units() - code_unit_offset); }
+
+ bool validate(size_t& valid_code_units) const;
+ bool validate() const
+ {
+ size_t valid_code_units;
+ return validate(valid_code_units);
+ }
+
+private:
+ u16 const* begin_ptr() const { return m_code_units.data(); }
+ u16 const* end_ptr() const { return begin_ptr() + m_code_units.size(); }
+
+ size_t calculate_length_in_code_points() const;
+
+ Span<u16 const> m_code_units;
+ mutable Optional<size_t> m_length_in_code_points;
+};
+
+}
+
+using AK::Utf16View;
diff --git a/Tests/AK/CMakeLists.txt b/Tests/AK/CMakeLists.txt
index 8459a55706..3598f480fe 100644
--- a/Tests/AK/CMakeLists.txt
+++ b/Tests/AK/CMakeLists.txt
@@ -61,6 +61,7 @@ set(AK_TEST_SOURCES
TestTypedTransfer.cpp
TestUFixedBigInt.cpp
TestURL.cpp
+ TestUtf16.cpp
TestUtf8.cpp
TestVariant.cpp
TestVector.cpp
diff --git a/Tests/AK/TestUtf16.cpp b/Tests/AK/TestUtf16.cpp
new file mode 100644
index 0000000000..e27699ff5b
--- /dev/null
+++ b/Tests/AK/TestUtf16.cpp
@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibTest/TestCase.h>
+
+#include <AK/Array.h>
+#include <AK/String.h>
+#include <AK/StringView.h>
+#include <AK/Types.h>
+#include <AK/Utf16View.h>
+
+TEST_CASE(decode_ascii)
+{
+ auto string = AK::utf8_to_utf16("Hello World!11"sv);
+ Utf16View view { string };
+
+ size_t valid_code_units = 0;
+ EXPECT(view.validate(valid_code_units));
+ EXPECT_EQ(valid_code_units, view.length_in_code_units());
+
+ auto expected = Array { (u32)72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 49, 49 };
+ EXPECT_EQ(expected.size(), view.length_in_code_points());
+
+ size_t i = 0;
+ for (u32 code_point : view) {
+ EXPECT_EQ(code_point, expected[i++]);
+ }
+ EXPECT_EQ(i, expected.size());
+}
+
+TEST_CASE(decode_utf8)
+{
+ auto string = AK::utf8_to_utf16("Привет, мир! 😀 γειά σου κόσμος こんにちは世界"sv);
+ Utf16View view { string };
+
+ size_t valid_code_units = 0;
+ EXPECT(view.validate(valid_code_units));
+ EXPECT_EQ(valid_code_units, view.length_in_code_units());
+
+ auto expected = Array { (u32)1055, 1088, 1080, 1074, 1077, 1090, 44, 32, 1084, 1080, 1088, 33, 32, 128512, 32, 947, 949, 953, 940, 32, 963, 959, 965, 32, 954, 972, 963, 956, 959, 962, 32, 12371, 12435, 12395, 12385, 12399, 19990, 30028 };
+ EXPECT_EQ(expected.size(), view.length_in_code_points());
+
+ size_t i = 0;
+ for (u32 code_point : view) {
+ EXPECT_EQ(code_point, expected[i++]);
+ }
+ EXPECT_EQ(i, expected.size());
+}
+
+TEST_CASE(encode_utf8)
+{
+ {
+ String utf8_string("Привет, мир! 😀 γειά σου κόσμος こんにちは世界");
+ auto string = AK::utf8_to_utf16(utf8_string);
+ Utf16View view { string };
+ EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes), utf8_string);
+ EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No), utf8_string);
+ }
+ {
+ auto encoded = Array { (u16)0xd83d };
+ Utf16View view { encoded };
+ EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes), "\xed\xa0\xbd"sv);
+ EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No), "\ufffd"sv);
+ }
+}
+
+TEST_CASE(decode_utf16)
+{
+ // Same string as the decode_utf8 test.
+ auto encoded = Array { (u16)0x041f, 0x0440, 0x0438, 0x0432, 0x0435, 0x0442, 0x002c, 0x0020, 0x043c, 0x0438, 0x0440, 0x0021, 0x0020, 0xd83d, 0xde00, 0x0020, 0x03b3, 0x03b5, 0x03b9, 0x03ac, 0x0020, 0x03c3, 0x03bf, 0x03c5, 0x0020, 0x03ba, 0x03cc, 0x03c3, 0x03bc, 0x03bf, 0x03c2, 0x0020, 0x3053, 0x3093, 0x306b, 0x3061, 0x306f, 0x4e16, 0x754c };
+
+ Utf16View view { encoded };
+ EXPECT_EQ(encoded.size(), view.length_in_code_units());
+
+ size_t valid_code_units = 0;
+ EXPECT(view.validate(valid_code_units));
+ EXPECT_EQ(valid_code_units, view.length_in_code_units());
+
+ auto expected = Array { (u32)1055, 1088, 1080, 1074, 1077, 1090, 44, 32, 1084, 1080, 1088, 33, 32, 128512, 32, 947, 949, 953, 940, 32, 963, 959, 965, 32, 954, 972, 963, 956, 959, 962, 32, 12371, 12435, 12395, 12385, 12399, 19990, 30028 };
+ EXPECT_EQ(expected.size(), view.length_in_code_points());
+
+ size_t i = 0;
+ for (u32 code_point : view) {
+ EXPECT_EQ(code_point, expected[i++]);
+ }
+ EXPECT_EQ(i, expected.size());
+}
+
+TEST_CASE(iterate_utf16)
+{
+ auto string = AK::utf8_to_utf16("Привет 😀"sv);
+ Utf16View view { string };
+ auto iterator = view.begin();
+
+ EXPECT(*iterator == 1055);
+ EXPECT(iterator.length_in_code_units() == 1);
+
+ EXPECT(++iterator != view.end());
+ EXPECT(*iterator == 1088);
+ EXPECT(iterator.length_in_code_units() == 1);
+
+ EXPECT(++iterator != view.end());
+ EXPECT(*iterator == 1080);
+ EXPECT(iterator.length_in_code_units() == 1);
+
+ EXPECT(++iterator != view.end());
+ EXPECT(*iterator == 1074);
+ EXPECT(iterator.length_in_code_units() == 1);
+
+ EXPECT(++iterator != view.end());
+ EXPECT(*iterator == 1077);
+ EXPECT(iterator.length_in_code_units() == 1);
+
+ EXPECT(++iterator != view.end());
+ EXPECT(*iterator == 1090);
+ EXPECT(iterator.length_in_code_units() == 1);
+
+ EXPECT(++iterator != view.end());
+ EXPECT(*iterator == 32);
+ EXPECT(iterator.length_in_code_units() == 1);
+
+ EXPECT(++iterator != view.end());
+ EXPECT(*iterator == 128512);
+ EXPECT(iterator.length_in_code_units() == 2);
+
+ EXPECT(++iterator == view.end());
+ EXPECT_CRASH("Dereferencing Utf16CodePointIterator which is at its end.", [&iterator] {
+ *iterator;
+ return Test::Crash::Failure::DidNotCrash;
+ });
+
+ EXPECT_CRASH("Incrementing Utf16CodePointIterator which is at its end.", [&iterator] {
+ ++iterator;
+ return Test::Crash::Failure::DidNotCrash;
+ });
+}
+
+TEST_CASE(validate_invalid_utf16)
+{
+ size_t valid_code_units = 0;
+ {
+ // Lonely high surrogate.
+ auto invalid = Array { (u16)0xd800 };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+
+ invalid = Array { (u16)0xdbff };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+ }
+ {
+ // Lonely low surrogate.
+ auto invalid = Array { (u16)0xdc00 };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+
+ invalid = Array { (u16)0xdfff };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+ }
+ {
+ // High surrogate followed by non-surrogate.
+ auto invalid = Array { (u16)0xd800, 0 };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+
+ invalid = Array { (u16)0xd800, 0xe000 };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+ }
+ {
+ // High surrogate followed by high surrogate.
+ auto invalid = Array { (u16)0xd800, 0xd800 };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+
+ invalid = Array { (u16)0xd800, 0xdbff };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 0);
+ }
+ {
+ // Valid UTF-16 followed by invalid code units.
+ auto invalid = Array { (u16)0x41, 0x41, 0xd800 };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 2);
+
+ invalid = Array { (u16)0x41, 0x41, 0xd800 };
+ EXPECT(!Utf16View(invalid).validate(valid_code_units));
+ EXPECT(valid_code_units == 2);
+ }
+}
+
+TEST_CASE(decode_invalid_utf16)
+{
+ {
+ // Lonely high surrogate.
+ auto invalid = Array { (u16)0x41, 0x42, 0xd800 };
+
+ Utf16View view { invalid };
+ EXPECT_EQ(invalid.size(), view.length_in_code_units());
+
+ auto expected = Array { (u32)0x41, 0x42, 0xfffd };
+ EXPECT_EQ(expected.size(), view.length_in_code_points());
+
+ size_t i = 0;
+ for (u32 code_point : view) {
+ EXPECT_EQ(code_point, expected[i++]);
+ }
+ EXPECT_EQ(i, expected.size());
+ }
+ {
+ // Lonely low surrogate.
+ auto invalid = Array { (u16)0x41, 0x42, 0xdc00 };
+
+ Utf16View view { invalid };
+ EXPECT_EQ(invalid.size(), view.length_in_code_units());
+
+ auto expected = Array { (u32)0x41, 0x42, 0xfffd };
+ EXPECT_EQ(expected.size(), view.length_in_code_points());
+
+ size_t i = 0;
+ for (u32 code_point : view) {
+ EXPECT_EQ(code_point, expected[i++]);
+ }
+ EXPECT_EQ(i, expected.size());
+ }
+ {
+ // High surrogate followed by non-surrogate.
+ auto invalid = Array { (u16)0x41, 0x42, 0xd800, 0 };
+
+ Utf16View view { invalid };
+ EXPECT_EQ(invalid.size(), view.length_in_code_units());
+
+ auto expected = Array { (u32)0x41, 0x42, 0xfffd, 0 };
+ EXPECT_EQ(expected.size(), view.length_in_code_points());
+
+ size_t i = 0;
+ for (u32 code_point : view) {
+ EXPECT_EQ(code_point, expected[i++]);
+ }
+ EXPECT_EQ(i, expected.size());
+ }
+ {
+ // High surrogate followed by high surrogate.
+ auto invalid = Array { (u16)0x41, 0x42, 0xd800, 0xd800 };
+
+ Utf16View view { invalid };
+ EXPECT_EQ(invalid.size(), view.length_in_code_units());
+
+ auto expected = Array { (u32)0x41, 0x42, 0xfffd, 0xfffd };
+ EXPECT_EQ(expected.size(), view.length_in_code_points());
+
+ size_t i = 0;
+ for (u32 code_point : view) {
+ EXPECT_EQ(code_point, expected[i++]);
+ }
+ EXPECT_EQ(i, expected.size());
+ }
+}
+
+TEST_CASE(substring_view)
+{
+ auto string = AK::utf8_to_utf16("Привет 😀"sv);
+ {
+ Utf16View view { string };
+ view = view.substring_view(7, 2);
+
+ EXPECT(view.length_in_code_units() == 2);
+ EXPECT_EQ(view.to_utf8(), "😀"sv);
+ }
+ {
+ Utf16View view { string };
+ view = view.substring_view(7, 1);
+
+ EXPECT(view.length_in_code_units() == 1);
+ EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes), "\xed\xa0\xbd"sv);
+ EXPECT_EQ(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No), "\ufffd"sv);
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-15 06:29:00 +0000