diff options
Diffstat (limited to 'Tests/LibWeb/test-web.cpp')
| -rw-r--r-- | Tests/LibWeb/test-web.cpp | 667 |
1 files changed, 667 insertions, 0 deletions
diff --git a/Tests/LibWeb/test-web.cpp b/Tests/LibWeb/test-web.cpp new file mode 100644 index 0000000000..1b963835de --- /dev/null +++ b/Tests/LibWeb/test-web.cpp @@ -0,0 +1,667 @@ +/* + * Copyright (c) 2020-2021, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/Function.h> +#include <AK/JsonObject.h> +#include <AK/JsonValue.h> +#include <AK/QuickSort.h> +#include <AK/URL.h> +#include <LibCore/ArgsParser.h> +#include <LibCore/DirIterator.h> +#include <LibCore/File.h> +#include <LibGUI/Application.h> +#include <LibGUI/BoxLayout.h> +#include <LibGUI/Widget.h> +#include <LibGUI/Window.h> +#include <LibJS/Interpreter.h> +#include <LibJS/Lexer.h> +#include <LibJS/Parser.h> +#include <LibJS/Runtime/Array.h> +#include <LibJS/Runtime/JSONObject.h> +#include <LibTest/Results.h> +#include <LibWeb/HTML/Parser/HTMLDocumentParser.h> +#include <LibWeb/InProcessWebView.h> +#include <LibWeb/Loader/ResourceLoader.h> +#include <signal.h> +#include <sys/time.h> + +#define TOP_LEVEL_TEST_NAME "__$$TOP_LEVEL$$__" + +struct ParserError { + JS::Parser::Error error; + String hint; +}; + +struct JSFileResult { + String name; + Optional<ParserError> error {}; + double time_taken { 0 }; + // A failed test takes precedence over a skipped test, which both have + // precedence over a passed test + Test::Result most_severe_test_result { Test::Result::Pass }; + Vector<Test::Suite> suites {}; + Vector<String> logged_messages {}; +}; + +Function<void(const URL&)> g_on_page_change; + +class TestRunnerObject final : public JS::Object { + JS_OBJECT(TestRunnerObject, JS::Object); + +public: + explicit TestRunnerObject(JS::GlobalObject&); + virtual void initialize(JS::GlobalObject&) override; + virtual ~TestRunnerObject() override; + +private: + JS_DECLARE_NATIVE_FUNCTION(change_page); +}; + +TestRunnerObject::TestRunnerObject(JS::GlobalObject& global_object) + : Object(*global_object.object_prototype()) +{ +} + +void TestRunnerObject::initialize(JS::GlobalObject& global_object) +{ + Object::initialize(global_object); + define_native_function("changePage", change_page, 1); +} + +TestRunnerObject::~TestRunnerObject() +{ +} + +JS_DEFINE_NATIVE_FUNCTION(TestRunnerObject::change_page) +{ + auto url = vm.argument(0).to_string(global_object); + if (vm.exception()) + return {}; + + if (g_on_page_change) + g_on_page_change(url); + + return JS::js_undefined(); +} + +class TestRunner { +public: + TestRunner(String web_test_root, String js_test_root, Web::InProcessWebView& page_view, bool print_times) + : m_web_test_root(move(web_test_root)) + , m_js_test_root(move(js_test_root)) + , m_print_times(print_times) + , m_page_view(page_view) + { + } + + void run(); + +private: + JSFileResult run_file_test(const String& test_path); + void print_file_result(const JSFileResult& file_result) const; + void print_test_results() const; + + String m_web_test_root; + String m_js_test_root; + bool m_print_times; + + double m_total_elapsed_time_in_ms { 0 }; + Test::Counts m_counts; + + RefPtr<Web::InProcessWebView> m_page_view; + + RefPtr<JS::Program> m_js_test_common; + RefPtr<JS::Program> m_web_test_common; +}; + +static void cleanup_and_exit() +{ + // Clear the taskbar progress. +#ifdef __serenity__ + fprintf(stderr, "\033]9;-1;\033\\"); +#endif + exit(1); +} + +#if 0 +static void handle_sigabrt(int) +{ + dbgln("test-web: SIGABRT received, cleaning up."); + cleanup_and_exit(); +} +#endif + +static double get_time_in_ms() +{ + struct timeval tv1; + auto return_code = gettimeofday(&tv1, nullptr); + VERIFY(return_code >= 0); + return static_cast<double>(tv1.tv_sec) * 1000.0 + static_cast<double>(tv1.tv_usec) / 1000.0; +} + +template<typename Callback> +void iterate_directory_recursively(const String& directory_path, Callback callback) +{ + Core::DirIterator directory_iterator(directory_path, Core::DirIterator::Flags::SkipDots); + + while (directory_iterator.has_next()) { + auto file_path = directory_iterator.next_full_path(); + if (Core::File::is_directory(file_path)) { + iterate_directory_recursively(file_path, callback); + } else { + callback(move(file_path)); + } + } +} + +static Vector<String> get_test_paths(const String& test_root) +{ + Vector<String> paths; + + iterate_directory_recursively(test_root, [&](const String& file_path) { + if (!file_path.ends_with("test-common.js") && !file_path.ends_with(".html") && !file_path.ends_with(".ts")) + paths.append(file_path); + }); + + quick_sort(paths); + + return paths; +} + +void TestRunner::run() +{ + size_t progress_counter = 0; + auto test_paths = get_test_paths(m_web_test_root); + + g_on_page_change = [this](auto& page_to_load) { + if (!page_to_load.is_valid()) { + printf("Invalid page URL (%s) on page change", page_to_load.to_string().characters()); + cleanup_and_exit(); + } + + VERIFY(m_page_view->document()); + + // We want to keep the same document since the interpreter is tied to the document, + // and we don't want to lose the test state. So, we just clear the document and + // give a new parser the existing document to work on. + m_page_view->document()->remove_all_children(); + + Web::LoadRequest request; + request.set_url(page_to_load); + + Web::ResourceLoader::the().load_sync( + request, + [&](auto data, auto&, auto) { + Web::HTML::HTMLDocumentParser parser(*m_page_view->document(), data, "utf-8"); + parser.run(page_to_load); + }, + [page_to_load](auto& error, auto) { + printf("Failed to load test page: %s (%s)", page_to_load.to_string().characters(), error.characters()); + cleanup_and_exit(); + }); + }; + + for (auto& path : test_paths) { + ++progress_counter; + print_file_result(run_file_test(path)); +#ifdef __serenity__ + fprintf(stderr, "\033]9;%zu;%zu;\033\\", progress_counter, test_paths.size()); +#endif + } + +#ifdef __serenity__ + fprintf(stderr, "\033]9;-1;\033\\"); +#endif + + print_test_results(); +} + +static Result<NonnullRefPtr<JS::Program>, ParserError> parse_file(const String& file_path) +{ + auto file = Core::File::construct(file_path); + auto result = file->open(Core::IODevice::ReadOnly); + if (!result) { + printf("Failed to open the following file: \"%s\"\n", file_path.characters()); + cleanup_and_exit(); + } + + auto contents = file->read_all(); + String test_file_string(reinterpret_cast<const char*>(contents.data()), contents.size()); + file->close(); + + auto parser = JS::Parser(JS::Lexer(test_file_string)); + auto program = parser.parse_program(); + + if (parser.has_errors()) { + auto error = parser.errors()[0]; + return Result<NonnullRefPtr<JS::Program>, ParserError>(ParserError { error, error.source_location_hint(test_file_string) }); + } + + return Result<NonnullRefPtr<JS::Program>, ParserError>(program); +} + +static Optional<JsonValue> get_test_results(JS::Interpreter& interpreter) +{ + auto result = interpreter.vm().get_variable("__TestResults__", interpreter.global_object()); + auto json_string = JS::JSONObject::stringify_impl(interpreter.global_object(), result, JS::js_undefined(), JS::js_undefined()); + + auto json = JsonValue::from_string(json_string); + if (!json.has_value()) + return {}; + + return json.value(); +} + +JSFileResult TestRunner::run_file_test(const String& test_path) +{ + double start_time = get_time_in_ms(); + VERIFY(m_page_view->document()); + auto& old_interpreter = m_page_view->document()->interpreter(); + + if (!m_js_test_common) { + auto result = parse_file(String::formatted("{}/test-common.js", m_js_test_root)); + if (result.is_error()) { + printf("Unable to parse %s/test-common.js\n", m_js_test_root.characters()); + printf("%s\n", result.error().error.to_string().characters()); + printf("%s\n", result.error().hint.characters()); + cleanup_and_exit(); + } + m_js_test_common = result.value(); + } + + if (!m_web_test_common) { + auto result = parse_file(String::formatted("{}/test-common.js", m_web_test_root)); + if (result.is_error()) { + printf("Unable to parse %s/test-common.js\n", m_web_test_root.characters()); + printf("%s\n", result.error().error.to_string().characters()); + printf("%s\n", result.error().hint.characters()); + cleanup_and_exit(); + } + m_web_test_common = result.value(); + } + + auto file_program = parse_file(test_path); + if (file_program.is_error()) + return { test_path, file_program.error() }; + + // Setup the test on the current page to get "__PageToLoad__". + old_interpreter.run(old_interpreter.global_object(), *m_web_test_common); + old_interpreter.run(old_interpreter.global_object(), *file_program.value()); + auto page_to_load = URL(old_interpreter.vm().get_variable("__PageToLoad__", old_interpreter.global_object()).as_string().string()); + if (!page_to_load.is_valid()) { + printf("Invalid page URL for %s", test_path.characters()); + cleanup_and_exit(); + } + + JSFileResult file_result; + + Web::LoadRequest request; + request.set_url(page_to_load); + + Web::ResourceLoader::the().load_sync( + request, + [&](auto data, auto&, auto) { + // Create a new parser and immediately get its document to replace the old interpreter. + auto document = Web::DOM::Document::create(); + Web::HTML::HTMLDocumentParser parser(document, data, "utf-8"); + auto& new_interpreter = parser.document().interpreter(); + + // FIXME: This is a hack while we're refactoring Interpreter/VM stuff. + JS::VM::InterpreterExecutionScope scope(new_interpreter); + + // Setup the test environment and call "__BeforeInitialPageLoad__" + new_interpreter.global_object().define_property( + "libweb_tester", + new_interpreter.heap().allocate<TestRunnerObject>(new_interpreter.global_object(), new_interpreter.global_object()), + JS::Attribute::Enumerable | JS::Attribute::Configurable); + new_interpreter.run(new_interpreter.global_object(), *m_js_test_common); + new_interpreter.run(new_interpreter.global_object(), *m_web_test_common); + new_interpreter.run(new_interpreter.global_object(), *file_program.value()); + + auto& before_initial_page_load = new_interpreter.vm().get_variable("__BeforeInitialPageLoad__", new_interpreter.global_object()).as_function(); + [[maybe_unused]] auto rc_before = new_interpreter.vm().call(before_initial_page_load, JS::js_undefined()); + if (new_interpreter.exception()) + new_interpreter.vm().clear_exception(); + + // Now parse the HTML page. + parser.run(page_to_load); + m_page_view->set_document(&parser.document()); + + // Finally run the test by calling "__AfterInitialPageLoad__" + auto& after_initial_page_load = new_interpreter.vm().get_variable("__AfterInitialPageLoad__", new_interpreter.global_object()).as_function(); + [[maybe_unused]] auto rc_after = new_interpreter.vm().call(after_initial_page_load, JS::js_undefined()); + if (new_interpreter.exception()) + new_interpreter.vm().clear_exception(); + + auto test_json = get_test_results(new_interpreter); + if (!test_json.has_value()) { + printf("Received malformed JSON from test \"%s\"\n", test_path.characters()); + cleanup_and_exit(); + } + + file_result = { test_path.substring(m_web_test_root.length() + 1, test_path.length() - m_web_test_root.length() - 1) }; + + // Collect logged messages + auto& arr = new_interpreter.vm().get_variable("__UserOutput__", new_interpreter.global_object()).as_array(); + for (auto& entry : arr.indexed_properties()) { + auto message = entry.value_and_attributes(&new_interpreter.global_object()).value; + file_result.logged_messages.append(message.to_string_without_side_effects()); + } + + test_json.value().as_object().for_each_member([&](const String& suite_name, const JsonValue& suite_value) { + Test::Suite suite { suite_name }; + + VERIFY(suite_value.is_object()); + + suite_value.as_object().for_each_member([&](const String& test_name, const JsonValue& test_value) { + Test::Case test { test_name, Test::Result::Fail, "" }; + + VERIFY(test_value.is_object()); + VERIFY(test_value.as_object().has("result")); + + auto result = test_value.as_object().get("result"); + VERIFY(result.is_string()); + auto result_string = result.as_string(); + if (result_string == "pass") { + test.result = Test::Result::Pass; + m_counts.tests_passed++; + } else if (result_string == "fail") { + test.result = Test::Result::Fail; + m_counts.tests_failed++; + suite.most_severe_test_result = Test::Result::Fail; + VERIFY(test_value.as_object().has("details")); + auto details = test_value.as_object().get("details"); + VERIFY(result.is_string()); + test.details = details.as_string(); + } else { + test.result = Test::Result::Skip; + if (suite.most_severe_test_result == Test::Result::Pass) + suite.most_severe_test_result = Test::Result::Skip; + m_counts.tests_skipped++; + } + + suite.tests.append(test); + }); + + if (suite.most_severe_test_result == Test::Result::Fail) { + m_counts.suites_failed++; + file_result.most_severe_test_result = Test::Result::Fail; + } else { + if (suite.most_severe_test_result == Test::Result::Skip && file_result.most_severe_test_result == Test::Result::Pass) + file_result.most_severe_test_result = Test::Result::Skip; + m_counts.suites_passed++; + } + + file_result.suites.append(suite); + }); + + m_counts.files_total++; + + file_result.time_taken = get_time_in_ms() - start_time; + m_total_elapsed_time_in_ms += file_result.time_taken; + }, + [page_to_load](auto& error, auto) { + printf("Failed to load test page: %s (%s)", page_to_load.to_string().characters(), error.characters()); + cleanup_and_exit(); + }); + + return file_result; +} + +enum Modifier { + BG_RED, + BG_GREEN, + FG_RED, + FG_GREEN, + FG_ORANGE, + FG_GRAY, + FG_BLACK, + FG_BOLD, + ITALIC, + CLEAR, +}; + +static void print_modifiers(Vector<Modifier> modifiers) +{ + for (auto& modifier : modifiers) { + auto code = [&]() -> String { + switch (modifier) { + case BG_RED: + return "\033[48;2;255;0;102m"; + case BG_GREEN: + return "\033[48;2;102;255;0m"; + case FG_RED: + return "\033[38;2;255;0;102m"; + case FG_GREEN: + return "\033[38;2;102;255;0m"; + case FG_ORANGE: + return "\033[38;2;255;102;0m"; + case FG_GRAY: + return "\033[38;2;135;139;148m"; + case FG_BLACK: + return "\033[30m"; + case FG_BOLD: + return "\033[1m"; + case ITALIC: + return "\033[3m"; + case CLEAR: + return "\033[0m"; + } + VERIFY_NOT_REACHED(); + }; + printf("%s", code().characters()); + } +} + +void TestRunner::print_file_result(const JSFileResult& file_result) const +{ + if (file_result.most_severe_test_result == Test::Result::Fail || file_result.error.has_value()) { + print_modifiers({ BG_RED, FG_BLACK, FG_BOLD }); + printf(" FAIL "); + print_modifiers({ CLEAR }); + } else { + if (m_print_times || file_result.most_severe_test_result != Test::Result::Pass) { + print_modifiers({ BG_GREEN, FG_BLACK, FG_BOLD }); + printf(" PASS "); + print_modifiers({ CLEAR }); + } else { + return; + } + } + + printf(" %s", file_result.name.characters()); + + if (m_print_times) { + print_modifiers({ CLEAR, ITALIC, FG_GRAY }); + if (file_result.time_taken < 1000) { + printf(" (%dms)\n", static_cast<int>(file_result.time_taken)); + } else { + printf(" (%.3fs)\n", file_result.time_taken / 1000.0); + } + print_modifiers({ CLEAR }); + } else { + printf("\n"); + } + + if (!file_result.logged_messages.is_empty()) { + print_modifiers({ FG_GRAY, FG_BOLD }); +#ifdef __serenity__ + printf(" âš Console output:\n"); +#else + // This emoji has a second invisible byte after it. The one above does not + printf(" âšī¸ Console output:\n"); +#endif + print_modifiers({ CLEAR, FG_GRAY }); + for (auto& message : file_result.logged_messages) + printf(" %s\n", message.characters()); + } + + if (file_result.error.has_value()) { + auto test_error = file_result.error.value(); + + print_modifiers({ FG_RED }); +#ifdef __serenity__ + printf(" â The file failed to parse\n\n"); +#else + // No invisible byte here, but the spacing still needs to be altered on the host + printf(" â The file failed to parse\n\n"); +#endif + print_modifiers({ FG_GRAY }); + for (auto& message : test_error.hint.split('\n', true)) { + printf(" %s\n", message.characters()); + } + print_modifiers({ FG_RED }); + printf(" %s\n\n", test_error.error.to_string().characters()); + + return; + } + + if (file_result.most_severe_test_result != Test::Result::Pass) { + for (auto& suite : file_result.suites) { + if (suite.most_severe_test_result == Test::Result::Pass) + continue; + + bool failed = suite.most_severe_test_result == Test::Result::Fail; + + print_modifiers({ FG_GRAY, FG_BOLD }); + + if (failed) { +#ifdef __serenity__ + printf(" â Suite: "); +#else + // No invisible byte here, but the spacing still needs to be altered on the host + printf(" â Suite: "); +#endif + } else { +#ifdef __serenity__ + printf(" â Suite: "); +#else + // This emoji has a second invisible byte after it. The one above does not + printf(" â ī¸ Suite: "); +#endif + } + + print_modifiers({ CLEAR, FG_GRAY }); + + if (suite.name == TOP_LEVEL_TEST_NAME) { + printf("<top-level>\n"); + } else { + printf("%s\n", suite.name.characters()); + } + print_modifiers({ CLEAR }); + + for (auto& test : suite.tests) { + if (test.result == Test::Result::Pass) + continue; + + print_modifiers({ FG_GRAY, FG_BOLD }); + printf(" Test: "); + if (test.result == Test::Result::Fail) { + print_modifiers({ CLEAR, FG_RED }); + printf("%s (failed):\n", test.name.characters()); + printf(" %s\n", test.details.characters()); + } else { + print_modifiers({ CLEAR, FG_ORANGE }); + printf("%s (skipped)\n", test.name.characters()); + } + print_modifiers({ CLEAR }); + } + } + } +} + +void TestRunner::print_test_results() const +{ + printf("\nTest Suites: "); + if (m_counts.suites_failed) { + print_modifiers({ FG_RED }); + printf("%d failed, ", m_counts.suites_failed); + print_modifiers({ CLEAR }); + } + if (m_counts.suites_passed) { + print_modifiers({ FG_GREEN }); + printf("%d passed, ", m_counts.suites_passed); + print_modifiers({ CLEAR }); + } + printf("%d total\n", m_counts.suites_failed + m_counts.suites_passed); + + printf("Tests: "); + if (m_counts.tests_failed) { + print_modifiers({ FG_RED }); + printf("%d failed, ", m_counts.tests_failed); + print_modifiers({ CLEAR }); + } + if (m_counts.tests_skipped) { + print_modifiers({ FG_ORANGE }); + printf("%d skipped, ", m_counts.tests_skipped); + print_modifiers({ CLEAR }); + } + if (m_counts.tests_passed) { + print_modifiers({ FG_GREEN }); + printf("%d passed, ", m_counts.tests_passed); + print_modifiers({ CLEAR }); + } + printf("%d total\n", m_counts.tests_failed + m_counts.tests_passed); + + printf("Files: %d total\n", m_counts.files_total); + + printf("Time: "); + if (m_total_elapsed_time_in_ms < 1000.0) { + printf("%dms\n\n", static_cast<int>(m_total_elapsed_time_in_ms)); + } else { + printf("%-.3fs\n\n", m_total_elapsed_time_in_ms / 1000.0); + } +} + +int main(int argc, char** argv) +{ + bool print_times = false; + bool show_window = false; + +#if 0 + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_flags = SA_NOCLDWAIT; + act.sa_handler = handle_sigabrt; + int rc = sigaction(SIGABRT, &act, nullptr); + if (rc < 0) { + perror("sigaction"); + return 1; + } +#endif + + Core::ArgsParser args_parser; + args_parser.add_option(print_times, "Show duration of each test", "show-time", 't'); + args_parser.add_option(show_window, "Show window while running tests", "window", 'w'); + args_parser.parse(argc, argv); + + auto app = GUI::Application::construct(argc, argv); + auto window = GUI::Window::construct(); + auto& main_widget = window->set_main_widget<GUI::Widget>(); + main_widget.set_fill_with_background_color(true); + main_widget.set_layout<GUI::VerticalBoxLayout>(); + auto& view = main_widget.add<Web::InProcessWebView>(); + + view.set_document(Web::DOM::Document::create()); + + if (show_window) { + window->set_title("LibWeb Test Window"); + window->resize(640, 480); + window->show(); + } + +#ifdef __serenity__ + TestRunner("/home/anon/web-tests", "/home/anon/js-tests", view, print_times).run(); +#else + char* serenity_source_dir = getenv("SERENITY_SOURCE_DIR"); + if (!serenity_source_dir) { + printf("test-web requires the SERENITY_SOURCE_DIR environment variable to be set"); + return 1; + } + TestRunner(String::formatted("{}/Userland/Libraries/LibWeb/Tests", serenity_source_dir), String::formatted("{}/Userland/Libraries/LibJS/Tests", serenity_source_dir), view, print_times).run(); +#endif + return 0; +} |