diff options
author | Itamar <itamar8910@gmail.com> | 2021-08-21 16:36:40 +0300 |
---|---|---|
committer | Andreas Kling <kling@serenityos.org> | 2021-08-21 22:09:56 +0200 |
commit | 606e05852f656e323a9cc8c5533045fb466cf1a1 (patch) | |
tree | 468814dbf3c8955b54b4b814f0798cea0f0f38e6 /Userland | |
parent | 7a4a32b112169a6ece0e0f53a94f0e40b63fd3d6 (diff) | |
download | serenity-606e05852f656e323a9cc8c5533045fb466cf1a1.zip |
LibCpp: Add lex_iterable() method to the Lexer
This allows us to collect the tokens iteratively instead of having to
lex the whole program and then get a tokens vector.
Diffstat (limited to 'Userland')
-rw-r--r-- | Userland/Libraries/LibCpp/Lexer.cpp | 17 | ||||
-rw-r--r-- | Userland/Libraries/LibCpp/Lexer.h | 9 |
2 files changed, 21 insertions, 5 deletions
diff --git a/Userland/Libraries/LibCpp/Lexer.cpp b/Userland/Libraries/LibCpp/Lexer.cpp index ea8367d0dc..85c61b8c6c 100644 --- a/Userland/Libraries/LibCpp/Lexer.cpp +++ b/Userland/Libraries/LibCpp/Lexer.cpp @@ -6,6 +6,7 @@ #include "Lexer.h" #include <AK/CharacterTypes.h> +#include <AK/Function.h> #include <AK/HashTable.h> #include <AK/StdLibExtras.h> #include <AK/String.h> @@ -205,15 +206,13 @@ static bool is_known_type(StringView const& string) return types.contains(string); } -Vector<Token> Lexer::lex() +void Lexer::lex_impl(Function<void(Token)> callback) { - Vector<Token> tokens; - size_t token_start_index = 0; Position token_start_position; auto emit_single_char_token = [&](auto type) { - tokens.empend(type, m_position, m_position, m_input.substring_view(m_index, 1)); + callback(Token(type, m_position, m_position, m_input.substring_view(m_index, 1))); consume(); }; @@ -224,7 +223,7 @@ Vector<Token> Lexer::lex() auto commit_token = [&](auto type) { if (m_options.ignore_whitespace && type == Token::Type::Whitespace) return; - tokens.empend(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index)); + callback(Token(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index))); }; auto emit_token_equals = [&](auto type, auto equals_type) { @@ -784,6 +783,14 @@ Vector<Token> Lexer::lex() dbgln("Unimplemented token character: {}", ch); emit_single_char_token(Token::Type::Unknown); } +} + +Vector<Token> Lexer::lex() +{ + Vector<Token> tokens; + lex_impl([&](auto token) { + tokens.append(move(token)); + }); return tokens; } diff --git a/Userland/Libraries/LibCpp/Lexer.h b/Userland/Libraries/LibCpp/Lexer.h index 88cd64d4a8..e3f958ac7e 100644 --- a/Userland/Libraries/LibCpp/Lexer.h +++ b/Userland/Libraries/LibCpp/Lexer.h @@ -17,12 +17,15 @@ public: explicit Lexer(StringView const&, size_t start_line = 0); Vector<Token> lex(); + template<typename Callback> + void lex_iterable(Callback); void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; } private: char peek(size_t offset = 0) const; char consume(); + void lex_impl(Function<void(Token)>); StringView m_input; size_t m_index { 0 }; @@ -34,4 +37,10 @@ private: } m_options; }; +template<typename Callback> +void Lexer::lex_iterable(Callback callback) +{ + return lex_impl(move(callback)); +} + } |