blob: 404b9eec4928760f87f28eb7879e48fc42c1627c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
/*
* Copyright (c) 2020-2021, the SerenityOS developers.
* Copyright (c) 2021-2023, Sam Atkins <atkinssj@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Optional.h>
#include <AK/StringView.h>
#include <AK/Types.h>
#include <AK/Utf8View.h>
#include <LibWeb/CSS/Parser/Token.h>
#include <LibWeb/Forward.h>
namespace Web::CSS::Parser {
class U32Twin {
public:
void set(size_t index, u32 value)
{
if (index == 0)
first = value;
if (index == 1)
second = value;
}
u32 first {};
u32 second {};
};
class U32Triplet {
public:
void set(size_t index, u32 value)
{
if (index == 0)
first = value;
if (index == 1)
second = value;
if (index == 2)
third = value;
}
U32Twin to_twin_12()
{
return { first, second };
}
U32Twin to_twin_23()
{
return { second, third };
}
u32 first {};
u32 second {};
u32 third {};
};
class Tokenizer {
public:
static ErrorOr<Vector<Token>> tokenize(StringView input, StringView encoding);
[[nodiscard]] static Token create_eof_token();
private:
explicit Tokenizer(String decoded_input);
[[nodiscard]] ErrorOr<Vector<Token>> tokenize();
size_t current_byte_offset() const;
ErrorOr<String> input_since(size_t offset) const;
[[nodiscard]] u32 next_code_point();
[[nodiscard]] u32 peek_code_point(size_t offset = 0) const;
[[nodiscard]] U32Twin peek_twin() const;
[[nodiscard]] U32Triplet peek_triplet() const;
[[nodiscard]] U32Twin start_of_input_stream_twin();
[[nodiscard]] U32Triplet start_of_input_stream_triplet();
[[nodiscard]] static Token create_new_token(Token::Type);
[[nodiscard]] static Token create_value_token(Token::Type, FlyString&& value, String&& representation);
[[nodiscard]] static Token create_value_token(Token::Type, u32 value, String&& representation);
[[nodiscard]] ErrorOr<Token> consume_a_token();
[[nodiscard]] ErrorOr<Token> consume_string_token(u32 ending_code_point);
[[nodiscard]] ErrorOr<Token> consume_a_numeric_token();
[[nodiscard]] ErrorOr<Token> consume_an_ident_like_token();
[[nodiscard]] Number consume_a_number();
[[nodiscard]] float convert_a_string_to_a_number(StringView);
[[nodiscard]] ErrorOr<FlyString> consume_an_ident_sequence();
[[nodiscard]] u32 consume_escaped_code_point();
[[nodiscard]] ErrorOr<Token> consume_a_url_token();
void consume_the_remnants_of_a_bad_url();
void consume_comments();
void consume_as_much_whitespace_as_possible();
void reconsume_current_input_code_point();
[[nodiscard]] static bool is_valid_escape_sequence(U32Twin);
[[nodiscard]] static bool would_start_an_ident_sequence(U32Triplet);
[[nodiscard]] static bool would_start_a_number(U32Triplet);
String m_decoded_input;
Utf8View m_utf8_view;
AK::Utf8CodePointIterator m_utf8_iterator;
AK::Utf8CodePointIterator m_prev_utf8_iterator;
Token::Position m_position;
Token::Position m_prev_position;
};
}
|
