LibCpp: Do lexing in the Preprocessor

We now call Preprocessor::process_and_lex() and pass the result to the
parser.

Doing the lexing in the preprocessor will allow us to maintain the
original position information of tokens after substituting definitions.

4 files changed, 30 insertions, 25 deletions

diff --git a/Userland/Libraries/LibCpp/Parser.cpp b/Userland/Libraries/LibCpp/Parser.cpp
index cbb8259ed2..da0d479fda 100644
--- a/Userland/Libraries/LibCpp/Parser.cpp
+++ b/Userland/Libraries/LibCpp/Parser.cpp
@@ -15,11 +15,11 @@
 
 namespace Cpp {
 
-Parser::Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& definitions)
+Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& definitions)
     : m_preprocessor_definitions(move(definitions))
     , m_filename(filename)
 {
-    initialize_program_tokens(program);
+    initialize_program_tokens(tokens);
     if constexpr (CPP_DEBUG) {
         dbgln("Tokens:");
         for (size_t i = 0; i < m_tokens.size(); ++i) {
@@ -28,10 +28,9 @@ Parser::Parser(const StringView& program, const String& filename, Preprocessor::
     }
 }
 
-void Parser::initialize_program_tokens(const StringView& program)
+void Parser::initialize_program_tokens(Vector<Token> const& tokens)
 {
-    Lexer lexer(program);
-    for (auto& token : lexer.lex()) {
+    for (auto& token : tokens) {
         if (token.type() == Token::Type::Whitespace)
             continue;
         if (token.type() == Token::Type::Identifier) {
@@ -1397,7 +1396,7 @@ bool Parser::match_ellipsis()
         return false;
     return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
 }
-void Parser::add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue& definition)
+void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition)
 {
     if (!definition.value.has_value())
         return;
diff --git a/Userland/Libraries/LibCpp/Parser.h b/Userland/Libraries/LibCpp/Parser.h
index 76d80bdd7b..b21eca15bc 100644
--- a/Userland/Libraries/LibCpp/Parser.h
+++ b/Userland/Libraries/LibCpp/Parser.h
@@ -18,7 +18,7 @@ class Parser final {
     AK_MAKE_NONCOPYABLE(Parser);
 
 public:
-    explicit Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& = {});
+    explicit Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& = {});
     ~Parser() = default;
 
     NonnullRefPtr<TranslationUnit> parse();
@@ -185,8 +185,8 @@ private:
     void consume_attribute_specification();
     void consume_access_specifier();
     bool match_ellipsis();
-    void initialize_program_tokens(const StringView& program);
-    void add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue&);
+    void initialize_program_tokens(Vector<Token> const& tokens);
+    void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&);
     Vector<StringView> parse_type_qualifiers();
     Vector<StringView> parse_function_qualifiers();
 
diff --git a/Userland/Libraries/LibCpp/Preprocessor.cpp b/Userland/Libraries/LibCpp/Preprocessor.cpp
index 9428f7ee0c..85d5f143e4 100644
--- a/Userland/Libraries/LibCpp/Preprocessor.cpp
+++ b/Userland/Libraries/LibCpp/Preprocessor.cpp
@@ -8,6 +8,7 @@
 #include <AK/Assertions.h>
 #include <AK/GenericLexer.h>
 #include <AK/StringBuilder.h>
+#include <LibCpp/Lexer.h>
 #include <ctype.h>
 
 namespace Cpp {
@@ -36,8 +37,9 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program)
     }
 }
 
-const String& Preprocessor::process()
+Vector<Token> Preprocessor::process_and_lex()
 {
+    Vector<Token> all_tokens;
     for (; m_line_index < m_lines.size(); ++m_line_index) {
         auto& line = m_lines[m_line_index];
 
@@ -51,14 +53,14 @@ const String& Preprocessor::process()
         }
 
         if (include_in_processed_text) {
-            m_builder.append(line);
+            for (auto& token : process_line(line)) {
+                if (token.type() != Token::Type::Whitespace)
+                    all_tokens.append(token);
+            }
         }
-
-        m_builder.append("\n");
     }
 
-    m_processed_text = m_builder.to_string();
-    return m_processed_text;
+    return all_tokens;
 }
 
 static void consume_whitespace(GenericLexer& lexer)
@@ -224,10 +226,14 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
     }
 }
 
-const String& Preprocessor::processed_text()
+Vector<Token> Preprocessor::process_line(const StringView& line)
 {
-    VERIFY(!m_processed_text.is_null());
-    return m_processed_text;
+    Lexer line_lexer { line, m_line_index };
+    auto tokens = line_lexer.lex();
+
+    // TODO: Go over tokens of line, do token substitution
+
+    return tokens;
 }
 
 };
diff --git a/Userland/Libraries/LibCpp/Preprocessor.h b/Userland/Libraries/LibCpp/Preprocessor.h
index c96bd8180a..e3dd45aa40 100644
--- a/Userland/Libraries/LibCpp/Preprocessor.h
+++ b/Userland/Libraries/LibCpp/Preprocessor.h
@@ -12,6 +12,7 @@
 #include <AK/String.h>
 #include <AK/StringView.h>
 #include <AK/Vector.h>
+#include <LibCpp/Token.h>
 
 namespace Cpp {
 
@@ -19,8 +20,7 @@ class Preprocessor {
 
 public:
     explicit Preprocessor(const String& filename, const StringView& program);
-    const String& process();
-    const String& processed_text();
+    Vector<Token> process_and_lex();
     Vector<StringView> included_paths() const { return m_included_paths; }
 
     struct DefinedValue {
@@ -38,13 +38,13 @@ public:
 
 private:
     using PreprocessorKeyword = StringView;
-    PreprocessorKeyword handle_preprocessor_line(const StringView&);
-    void handle_preprocessor_keyword(const StringView& keyword, GenericLexer& line_lexer);
+    PreprocessorKeyword handle_preprocessor_line(StringView const&);
+    void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
+    Vector<Token> process_line(StringView const& line);
 
+    String m_filename;
+    String m_program;
     Definitions m_definitions;
-    const String m_filename;
-    const StringView m_program;
-    StringBuilder m_builder;
     Vector<StringView> m_lines;
     size_t m_line_index { 0 };
     size_t m_current_depth { 0 };