summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Tests/LibCpp/test-cpp-parser.cpp3
-rw-r--r--Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp4
-rw-r--r--Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp4
-rw-r--r--Userland/Libraries/LibCpp/Parser.cpp11
-rw-r--r--Userland/Libraries/LibCpp/Parser.h6
-rw-r--r--Userland/Libraries/LibCpp/Preprocessor.cpp24
-rw-r--r--Userland/Libraries/LibCpp/Preprocessor.h14
-rw-r--r--Userland/Utilities/PreprocessorTest.cpp5
-rw-r--r--Userland/Utilities/cpp-parser.cpp10
9 files changed, 42 insertions, 39 deletions
diff --git a/Tests/LibCpp/test-cpp-parser.cpp b/Tests/LibCpp/test-cpp-parser.cpp
index 907818ea4a..157fd9233e 100644
--- a/Tests/LibCpp/test-cpp-parser.cpp
+++ b/Tests/LibCpp/test-cpp-parser.cpp
@@ -43,7 +43,8 @@ TEST_CASE(test_regression)
auto target_ast = read_all(ast_file_path);
StringView source_view(source);
- ::Cpp::Parser parser(source_view, file_path);
+ Cpp::Preprocessor preprocessor(file_path, source_view);
+ Cpp::Parser parser(preprocessor.process_and_lex(), file_path);
auto root = parser.parse();
EXPECT(parser.errors().is_empty());
diff --git a/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp b/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp
index 20d43636b8..fd07b4f370 100644
--- a/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp
+++ b/Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp
@@ -569,7 +569,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
document_data->m_preprocessor = make<Preprocessor>(document_data->m_filename, document_data->text());
document_data->preprocessor().set_ignore_unsupported_keywords(true);
document_data->preprocessor().set_keep_include_statements(true);
- document_data->preprocessor().process();
+ auto tokens = document_data->preprocessor().process_and_lex();
Preprocessor::Definitions preprocessor_definitions;
for (auto item : document_data->preprocessor().definitions())
@@ -590,7 +590,7 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
preprocessor_definitions.set(move(item.key), move(item.value));
}
- document_data->m_parser = make<Parser>(document_data->preprocessor().processed_text(), filename, move(preprocessor_definitions));
+ document_data->m_parser = make<Parser>(move(tokens), filename, move(preprocessor_definitions));
auto root = document_data->parser().parse();
diff --git a/Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp b/Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp
index 3b69ee417c..275b276fd7 100644
--- a/Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp
+++ b/Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp
@@ -134,14 +134,14 @@ void test_complete_includes()
add_file(filedb, "sample_header.h");
CppComprehensionEngine autocomplete(filedb);
- auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 23 });
+ auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 22 });
if (suggestions.size() != 1)
FAIL(project include - bad size);
if (suggestions[0].completion != "sample_header.h")
FAIL("project include - wrong results");
- suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 19 });
+ suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 1, 18 });
if (suggestions.size() != 1)
FAIL(global include - bad size);
diff --git a/Userland/Libraries/LibCpp/Parser.cpp b/Userland/Libraries/LibCpp/Parser.cpp
index cbb8259ed2..da0d479fda 100644
--- a/Userland/Libraries/LibCpp/Parser.cpp
+++ b/Userland/Libraries/LibCpp/Parser.cpp
@@ -15,11 +15,11 @@
namespace Cpp {
-Parser::Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& definitions)
+Parser::Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& definitions)
: m_preprocessor_definitions(move(definitions))
, m_filename(filename)
{
- initialize_program_tokens(program);
+ initialize_program_tokens(tokens);
if constexpr (CPP_DEBUG) {
dbgln("Tokens:");
for (size_t i = 0; i < m_tokens.size(); ++i) {
@@ -28,10 +28,9 @@ Parser::Parser(const StringView& program, const String& filename, Preprocessor::
}
}
-void Parser::initialize_program_tokens(const StringView& program)
+void Parser::initialize_program_tokens(Vector<Token> const& tokens)
{
- Lexer lexer(program);
- for (auto& token : lexer.lex()) {
+ for (auto& token : tokens) {
if (token.type() == Token::Type::Whitespace)
continue;
if (token.type() == Token::Type::Identifier) {
@@ -1397,7 +1396,7 @@ bool Parser::match_ellipsis()
return false;
return peek().type() == Token::Type::Dot && peek(1).type() == Token::Type::Dot && peek(2).type() == Token::Type::Dot;
}
-void Parser::add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue& definition)
+void Parser::add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue& definition)
{
if (!definition.value.has_value())
return;
diff --git a/Userland/Libraries/LibCpp/Parser.h b/Userland/Libraries/LibCpp/Parser.h
index 76d80bdd7b..b21eca15bc 100644
--- a/Userland/Libraries/LibCpp/Parser.h
+++ b/Userland/Libraries/LibCpp/Parser.h
@@ -18,7 +18,7 @@ class Parser final {
AK_MAKE_NONCOPYABLE(Parser);
public:
- explicit Parser(const StringView& program, const String& filename, Preprocessor::Definitions&& = {});
+ explicit Parser(Vector<Token> const& tokens, const String& filename, Preprocessor::Definitions&& = {});
~Parser() = default;
NonnullRefPtr<TranslationUnit> parse();
@@ -185,8 +185,8 @@ private:
void consume_attribute_specification();
void consume_access_specifier();
bool match_ellipsis();
- void initialize_program_tokens(const StringView& program);
- void add_tokens_for_preprocessor(Token& replaced_token, Preprocessor::DefinedValue&);
+ void initialize_program_tokens(Vector<Token> const& tokens);
+ void add_tokens_for_preprocessor(Token const& replaced_token, Preprocessor::DefinedValue&);
Vector<StringView> parse_type_qualifiers();
Vector<StringView> parse_function_qualifiers();
diff --git a/Userland/Libraries/LibCpp/Preprocessor.cpp b/Userland/Libraries/LibCpp/Preprocessor.cpp
index 9428f7ee0c..85d5f143e4 100644
--- a/Userland/Libraries/LibCpp/Preprocessor.cpp
+++ b/Userland/Libraries/LibCpp/Preprocessor.cpp
@@ -8,6 +8,7 @@
#include <AK/Assertions.h>
#include <AK/GenericLexer.h>
#include <AK/StringBuilder.h>
+#include <LibCpp/Lexer.h>
#include <ctype.h>
namespace Cpp {
@@ -36,8 +37,9 @@ Preprocessor::Preprocessor(const String& filename, const StringView& program)
}
}
-const String& Preprocessor::process()
+Vector<Token> Preprocessor::process_and_lex()
{
+ Vector<Token> all_tokens;
for (; m_line_index < m_lines.size(); ++m_line_index) {
auto& line = m_lines[m_line_index];
@@ -51,14 +53,14 @@ const String& Preprocessor::process()
}
if (include_in_processed_text) {
- m_builder.append(line);
+ for (auto& token : process_line(line)) {
+ if (token.type() != Token::Type::Whitespace)
+ all_tokens.append(token);
+ }
}
-
- m_builder.append("\n");
}
- m_processed_text = m_builder.to_string();
- return m_processed_text;
+ return all_tokens;
}
static void consume_whitespace(GenericLexer& lexer)
@@ -224,10 +226,14 @@ void Preprocessor::handle_preprocessor_keyword(const StringView& keyword, Generi
}
}
-const String& Preprocessor::processed_text()
+Vector<Token> Preprocessor::process_line(const StringView& line)
{
- VERIFY(!m_processed_text.is_null());
- return m_processed_text;
+ Lexer line_lexer { line, m_line_index };
+ auto tokens = line_lexer.lex();
+
+ // TODO: Go over tokens of line, do token substitution
+
+ return tokens;
}
};
diff --git a/Userland/Libraries/LibCpp/Preprocessor.h b/Userland/Libraries/LibCpp/Preprocessor.h
index c96bd8180a..e3dd45aa40 100644
--- a/Userland/Libraries/LibCpp/Preprocessor.h
+++ b/Userland/Libraries/LibCpp/Preprocessor.h
@@ -12,6 +12,7 @@
#include <AK/String.h>
#include <AK/StringView.h>
#include <AK/Vector.h>
+#include <LibCpp/Token.h>
namespace Cpp {
@@ -19,8 +20,7 @@ class Preprocessor {
public:
explicit Preprocessor(const String& filename, const StringView& program);
- const String& process();
- const String& processed_text();
+ Vector<Token> process_and_lex();
Vector<StringView> included_paths() const { return m_included_paths; }
struct DefinedValue {
@@ -38,13 +38,13 @@ public:
private:
using PreprocessorKeyword = StringView;
- PreprocessorKeyword handle_preprocessor_line(const StringView&);
- void handle_preprocessor_keyword(const StringView& keyword, GenericLexer& line_lexer);
+ PreprocessorKeyword handle_preprocessor_line(StringView const&);
+ void handle_preprocessor_keyword(StringView const& keyword, GenericLexer& line_lexer);
+ Vector<Token> process_line(StringView const& line);
+ String m_filename;
+ String m_program;
Definitions m_definitions;
- const String m_filename;
- const StringView m_program;
- StringBuilder m_builder;
Vector<StringView> m_lines;
size_t m_line_index { 0 };
size_t m_current_depth { 0 };
diff --git a/Userland/Utilities/PreprocessorTest.cpp b/Userland/Utilities/PreprocessorTest.cpp
index 1e9594dedc..0069756681 100644
--- a/Userland/Utilities/PreprocessorTest.cpp
+++ b/Userland/Utilities/PreprocessorTest.cpp
@@ -16,5 +16,8 @@ int main(int, char**)
}
auto content = file->read_all();
Cpp::Preprocessor cpp("other.h", StringView { content });
- dbgln("{}", cpp.process());
+ auto tokens = cpp.process_and_lex();
+ for (auto& token : tokens) {
+ dbgln("{}", token.to_string());
+ }
}
diff --git a/Userland/Utilities/cpp-parser.cpp b/Userland/Utilities/cpp-parser.cpp
index 471a2ef7d1..5ce02563cf 100644
--- a/Userland/Utilities/cpp-parser.cpp
+++ b/Userland/Utilities/cpp-parser.cpp
@@ -13,9 +13,7 @@ int main(int argc, char** argv)
Core::ArgsParser args_parser;
const char* path = nullptr;
bool tokens_mode = false;
- bool preprocess_mode = false;
args_parser.add_option(tokens_mode, "Print Tokens", "tokens", 'T');
- args_parser.add_option(preprocess_mode, "Print Preprocessed source", "preprocessed", 'P');
args_parser.add_positional_argument(path, "Cpp File", "cpp-file", Core::ArgsParser::Required::No);
args_parser.parse(argc, argv);
@@ -30,13 +28,9 @@ int main(int argc, char** argv)
StringView content_view(content);
::Cpp::Preprocessor processor(path, content_view);
- auto preprocessed_content = processor.process();
- if (preprocess_mode) {
- outln(preprocessed_content);
- return 0;
- }
+ auto tokens = processor.process_and_lex();
- ::Cpp::Parser parser(preprocessed_content, path);
+ ::Cpp::Parser parser(tokens, path);
if (tokens_mode) {
parser.print_tokens();
return 0;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-15 16:53:48 +0000