1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
/*
* Copyright (c) 2020, Stephan Unverwerth <s.unverwerth@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include "Token.h"
#include <AK/HashMap.h>
#include <AK/String.h>
#include <AK/StringView.h>
namespace JS {
class Lexer {
public:
explicit Lexer(StringView source, StringView filename = "(unknown)", size_t line_number = 1, size_t line_column = 0);
Token next();
const StringView& source() const { return m_source; };
const StringView& filename() const { return m_filename; };
void disallow_html_comments() { m_allow_html_comments = false; };
Token force_slash_as_regex();
private:
void consume();
bool consume_exponent();
bool consume_octal_number();
bool consume_hexadecimal_number();
bool consume_binary_number();
bool consume_decimal_number();
bool is_unicode_character() const;
u32 current_code_point() const;
bool is_eof() const;
bool is_line_terminator() const;
bool is_whitespace() const;
Optional<u32> is_identifier_unicode_escape(size_t& identifier_length) const;
Optional<u32> is_identifier_start(size_t& identifier_length) const;
Optional<u32> is_identifier_middle(size_t& identifier_length) const;
bool is_line_comment_start(bool line_has_token_yet) const;
bool is_block_comment_start() const;
bool is_block_comment_end() const;
bool is_numeric_literal_start() const;
bool match(char, char) const;
bool match(char, char, char) const;
bool match(char, char, char, char) const;
template<typename Callback>
bool match_numeric_literal_separator_followed_by(Callback) const;
bool slash_means_division() const;
TokenType consume_regex_literal();
StringView m_source;
size_t m_position { 0 };
Token m_current_token;
char m_current_char { 0 };
bool m_eof { false };
StringView m_filename;
size_t m_line_number { 1 };
size_t m_line_column { 0 };
bool m_regex_is_in_character_class { false };
struct TemplateState {
bool in_expr;
u8 open_bracket_count;
};
Vector<TemplateState> m_template_states;
bool m_allow_html_comments { true };
static HashMap<FlyString, TokenType> s_keywords;
static HashMap<String, TokenType> s_three_char_tokens;
static HashMap<String, TokenType> s_two_char_tokens;
static HashMap<char, TokenType> s_single_char_tokens;
struct ParsedIdentifiers : public RefCounted<ParsedIdentifiers> {
// Resolved identifiers must be kept alive for the duration of the parsing stage, otherwise
// the only references to these strings are deleted by the Token destructor.
HashTable<FlyString> identifiers;
};
RefPtr<ParsedIdentifiers> m_parsed_identifiers;
};
}
|
