Tests: Move Userland/Utilities/test-web to Tests/LibWeb

1 files changed, 667 insertions, 0 deletions

diff --git a/Tests/LibWeb/test-web.cpp b/Tests/LibWeb/test-web.cpp
new file mode 100644
index 0000000000..1b963835de
--- /dev/null
+++ b/Tests/LibWeb/test-web.cpp
@@ -0,0 +1,667 @@
+/*
+ * Copyright (c) 2020-2021, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Function.h>
+#include <AK/JsonObject.h>
+#include <AK/JsonValue.h>
+#include <AK/QuickSort.h>
+#include <AK/URL.h>
+#include <LibCore/ArgsParser.h>
+#include <LibCore/DirIterator.h>
+#include <LibCore/File.h>
+#include <LibGUI/Application.h>
+#include <LibGUI/BoxLayout.h>
+#include <LibGUI/Widget.h>
+#include <LibGUI/Window.h>
+#include <LibJS/Interpreter.h>
+#include <LibJS/Lexer.h>
+#include <LibJS/Parser.h>
+#include <LibJS/Runtime/Array.h>
+#include <LibJS/Runtime/JSONObject.h>
+#include <LibTest/Results.h>
+#include <LibWeb/HTML/Parser/HTMLDocumentParser.h>
+#include <LibWeb/InProcessWebView.h>
+#include <LibWeb/Loader/ResourceLoader.h>
+#include <signal.h>
+#include <sys/time.h>
+
+#define TOP_LEVEL_TEST_NAME "__$$TOP_LEVEL$$__"
+
+struct ParserError {
+    JS::Parser::Error error;
+    String hint;
+};
+
+struct JSFileResult {
+    String name;
+    Optional<ParserError> error {};
+    double time_taken { 0 };
+    // A failed test takes precedence over a skipped test, which both have
+    // precedence over a passed test
+    Test::Result most_severe_test_result { Test::Result::Pass };
+    Vector<Test::Suite> suites {};
+    Vector<String> logged_messages {};
+};
+
+Function<void(const URL&)> g_on_page_change;
+
+class TestRunnerObject final : public JS::Object {
+    JS_OBJECT(TestRunnerObject, JS::Object);
+
+public:
+    explicit TestRunnerObject(JS::GlobalObject&);
+    virtual void initialize(JS::GlobalObject&) override;
+    virtual ~TestRunnerObject() override;
+
+private:
+    JS_DECLARE_NATIVE_FUNCTION(change_page);
+};
+
+TestRunnerObject::TestRunnerObject(JS::GlobalObject& global_object)
+    : Object(*global_object.object_prototype())
+{
+}
+
+void TestRunnerObject::initialize(JS::GlobalObject& global_object)
+{
+    Object::initialize(global_object);
+    define_native_function("changePage", change_page, 1);
+}
+
+TestRunnerObject::~TestRunnerObject()
+{
+}
+
+JS_DEFINE_NATIVE_FUNCTION(TestRunnerObject::change_page)
+{
+    auto url = vm.argument(0).to_string(global_object);
+    if (vm.exception())
+        return {};
+
+    if (g_on_page_change)
+        g_on_page_change(url);
+
+    return JS::js_undefined();
+}
+
+class TestRunner {
+public:
+    TestRunner(String web_test_root, String js_test_root, Web::InProcessWebView& page_view, bool print_times)
+        : m_web_test_root(move(web_test_root))
+        , m_js_test_root(move(js_test_root))
+        , m_print_times(print_times)
+        , m_page_view(page_view)
+    {
+    }
+
+    void run();
+
+private:
+    JSFileResult run_file_test(const String& test_path);
+    void print_file_result(const JSFileResult& file_result) const;
+    void print_test_results() const;
+
+    String m_web_test_root;
+    String m_js_test_root;
+    bool m_print_times;
+
+    double m_total_elapsed_time_in_ms { 0 };
+    Test::Counts m_counts;
+
+    RefPtr<Web::InProcessWebView> m_page_view;
+
+    RefPtr<JS::Program> m_js_test_common;
+    RefPtr<JS::Program> m_web_test_common;
+};
+
+static void cleanup_and_exit()
+{
+    // Clear the taskbar progress.
+#ifdef __serenity__
+    fprintf(stderr, "\033]9;-1;\033\\");
+#endif
+    exit(1);
+}
+
+#if 0
+static void handle_sigabrt(int)
+{
+    dbgln("test-web: SIGABRT received, cleaning up.");
+    cleanup_and_exit();
+}
+#endif
+
+static double get_time_in_ms()
+{
+    struct timeval tv1;
+    auto return_code = gettimeofday(&tv1, nullptr);
+    VERIFY(return_code >= 0);
+    return static_cast<double>(tv1.tv_sec) * 1000.0 + static_cast<double>(tv1.tv_usec) / 1000.0;
+}
+
+template<typename Callback>
+void iterate_directory_recursively(const String& directory_path, Callback callback)
+{
+    Core::DirIterator directory_iterator(directory_path, Core::DirIterator::Flags::SkipDots);
+
+    while (directory_iterator.has_next()) {
+        auto file_path = directory_iterator.next_full_path();
+        if (Core::File::is_directory(file_path)) {
+            iterate_directory_recursively(file_path, callback);
+        } else {
+            callback(move(file_path));
+        }
+    }
+}
+
+static Vector<String> get_test_paths(const String& test_root)
+{
+    Vector<String> paths;
+
+    iterate_directory_recursively(test_root, [&](const String& file_path) {
+        if (!file_path.ends_with("test-common.js") && !file_path.ends_with(".html") && !file_path.ends_with(".ts"))
+            paths.append(file_path);
+    });
+
+    quick_sort(paths);
+
+    return paths;
+}
+
+void TestRunner::run()
+{
+    size_t progress_counter = 0;
+    auto test_paths = get_test_paths(m_web_test_root);
+
+    g_on_page_change = [this](auto& page_to_load) {
+        if (!page_to_load.is_valid()) {
+            printf("Invalid page URL (%s) on page change", page_to_load.to_string().characters());
+            cleanup_and_exit();
+        }
+
+        VERIFY(m_page_view->document());
+
+        // We want to keep the same document since the interpreter is tied to the document,
+        // and we don't want to lose the test state. So, we just clear the document and
+        // give a new parser the existing document to work on.
+        m_page_view->document()->remove_all_children();
+
+        Web::LoadRequest request;
+        request.set_url(page_to_load);
+
+        Web::ResourceLoader::the().load_sync(
+            request,
+            [&](auto data, auto&, auto) {
+                Web::HTML::HTMLDocumentParser parser(*m_page_view->document(), data, "utf-8");
+                parser.run(page_to_load);
+            },
+            [page_to_load](auto& error, auto) {
+                printf("Failed to load test page: %s (%s)", page_to_load.to_string().characters(), error.characters());
+                cleanup_and_exit();
+            });
+    };
+
+    for (auto& path : test_paths) {
+        ++progress_counter;
+        print_file_result(run_file_test(path));
+#ifdef __serenity__
+        fprintf(stderr, "\033]9;%zu;%zu;\033\\", progress_counter, test_paths.size());
+#endif
+    }
+
+#ifdef __serenity__
+    fprintf(stderr, "\033]9;-1;\033\\");
+#endif
+
+    print_test_results();
+}
+
+static Result<NonnullRefPtr<JS::Program>, ParserError> parse_file(const String& file_path)
+{
+    auto file = Core::File::construct(file_path);
+    auto result = file->open(Core::IODevice::ReadOnly);
+    if (!result) {
+        printf("Failed to open the following file: \"%s\"\n", file_path.characters());
+        cleanup_and_exit();
+    }
+
+    auto contents = file->read_all();
+    String test_file_string(reinterpret_cast<const char*>(contents.data()), contents.size());
+    file->close();
+
+    auto parser = JS::Parser(JS::Lexer(test_file_string));
+    auto program = parser.parse_program();
+
+    if (parser.has_errors()) {
+        auto error = parser.errors()[0];
+        return Result<NonnullRefPtr<JS::Program>, ParserError>(ParserError { error, error.source_location_hint(test_file_string) });
+    }
+
+    return Result<NonnullRefPtr<JS::Program>, ParserError>(program);
+}
+
+static Optional<JsonValue> get_test_results(JS::Interpreter& interpreter)
+{
+    auto result = interpreter.vm().get_variable("__TestResults__", interpreter.global_object());
+    auto json_string = JS::JSONObject::stringify_impl(interpreter.global_object(), result, JS::js_undefined(), JS::js_undefined());
+
+    auto json = JsonValue::from_string(json_string);
+    if (!json.has_value())
+        return {};
+
+    return json.value();
+}
+
+JSFileResult TestRunner::run_file_test(const String& test_path)
+{
+    double start_time = get_time_in_ms();
+    VERIFY(m_page_view->document());
+    auto& old_interpreter = m_page_view->document()->interpreter();
+
+    if (!m_js_test_common) {
+        auto result = parse_file(String::formatted("{}/test-common.js", m_js_test_root));
+        if (result.is_error()) {
+            printf("Unable to parse %s/test-common.js\n", m_js_test_root.characters());
+            printf("%s\n", result.error().error.to_string().characters());
+            printf("%s\n", result.error().hint.characters());
+            cleanup_and_exit();
+        }
+        m_js_test_common = result.value();
+    }
+
+    if (!m_web_test_common) {
+        auto result = parse_file(String::formatted("{}/test-common.js", m_web_test_root));
+        if (result.is_error()) {
+            printf("Unable to parse %s/test-common.js\n", m_web_test_root.characters());
+            printf("%s\n", result.error().error.to_string().characters());
+            printf("%s\n", result.error().hint.characters());
+            cleanup_and_exit();
+        }
+        m_web_test_common = result.value();
+    }
+
+    auto file_program = parse_file(test_path);
+    if (file_program.is_error())
+        return { test_path, file_program.error() };
+
+    // Setup the test on the current page to get "__PageToLoad__".
+    old_interpreter.run(old_interpreter.global_object(), *m_web_test_common);
+    old_interpreter.run(old_interpreter.global_object(), *file_program.value());
+    auto page_to_load = URL(old_interpreter.vm().get_variable("__PageToLoad__", old_interpreter.global_object()).as_string().string());
+    if (!page_to_load.is_valid()) {
+        printf("Invalid page URL for %s", test_path.characters());
+        cleanup_and_exit();
+    }
+
+    JSFileResult file_result;
+
+    Web::LoadRequest request;
+    request.set_url(page_to_load);
+
+    Web::ResourceLoader::the().load_sync(
+        request,
+        [&](auto data, auto&, auto) {
+            // Create a new parser and immediately get its document to replace the old interpreter.
+            auto document = Web::DOM::Document::create();
+            Web::HTML::HTMLDocumentParser parser(document, data, "utf-8");
+            auto& new_interpreter = parser.document().interpreter();
+
+            // FIXME: This is a hack while we're refactoring Interpreter/VM stuff.
+            JS::VM::InterpreterExecutionScope scope(new_interpreter);
+
+            // Setup the test environment and call "__BeforeInitialPageLoad__"
+            new_interpreter.global_object().define_property(
+                "libweb_tester",
+                new_interpreter.heap().allocate<TestRunnerObject>(new_interpreter.global_object(), new_interpreter.global_object()),
+                JS::Attribute::Enumerable | JS::Attribute::Configurable);
+            new_interpreter.run(new_interpreter.global_object(), *m_js_test_common);
+            new_interpreter.run(new_interpreter.global_object(), *m_web_test_common);
+            new_interpreter.run(new_interpreter.global_object(), *file_program.value());
+
+            auto& before_initial_page_load = new_interpreter.vm().get_variable("__BeforeInitialPageLoad__", new_interpreter.global_object()).as_function();
+            [[maybe_unused]] auto rc_before = new_interpreter.vm().call(before_initial_page_load, JS::js_undefined());
+            if (new_interpreter.exception())
+                new_interpreter.vm().clear_exception();
+
+            // Now parse the HTML page.
+            parser.run(page_to_load);
+            m_page_view->set_document(&parser.document());
+
+            // Finally run the test by calling "__AfterInitialPageLoad__"
+            auto& after_initial_page_load = new_interpreter.vm().get_variable("__AfterInitialPageLoad__", new_interpreter.global_object()).as_function();
+            [[maybe_unused]] auto rc_after = new_interpreter.vm().call(after_initial_page_load, JS::js_undefined());
+            if (new_interpreter.exception())
+                new_interpreter.vm().clear_exception();
+
+            auto test_json = get_test_results(new_interpreter);
+            if (!test_json.has_value()) {
+                printf("Received malformed JSON from test \"%s\"\n", test_path.characters());
+                cleanup_and_exit();
+            }
+
+            file_result = { test_path.substring(m_web_test_root.length() + 1, test_path.length() - m_web_test_root.length() - 1) };
+
+            // Collect logged messages
+            auto& arr = new_interpreter.vm().get_variable("__UserOutput__", new_interpreter.global_object()).as_array();
+            for (auto& entry : arr.indexed_properties()) {
+                auto message = entry.value_and_attributes(&new_interpreter.global_object()).value;
+                file_result.logged_messages.append(message.to_string_without_side_effects());
+            }
+
+            test_json.value().as_object().for_each_member([&](const String& suite_name, const JsonValue& suite_value) {
+                Test::Suite suite { suite_name };
+
+                VERIFY(suite_value.is_object());
+
+                suite_value.as_object().for_each_member([&](const String& test_name, const JsonValue& test_value) {
+                    Test::Case test { test_name, Test::Result::Fail, "" };
+
+                    VERIFY(test_value.is_object());
+                    VERIFY(test_value.as_object().has("result"));
+
+                    auto result = test_value.as_object().get("result");
+                    VERIFY(result.is_string());
+                    auto result_string = result.as_string();
+                    if (result_string == "pass") {
+                        test.result = Test::Result::Pass;
+                        m_counts.tests_passed++;
+                    } else if (result_string == "fail") {
+                        test.result = Test::Result::Fail;
+                        m_counts.tests_failed++;
+                        suite.most_severe_test_result = Test::Result::Fail;
+                        VERIFY(test_value.as_object().has("details"));
+                        auto details = test_value.as_object().get("details");
+                        VERIFY(result.is_string());
+                        test.details = details.as_string();
+                    } else {
+                        test.result = Test::Result::Skip;
+                        if (suite.most_severe_test_result == Test::Result::Pass)
+                            suite.most_severe_test_result = Test::Result::Skip;
+                        m_counts.tests_skipped++;
+                    }
+
+                    suite.tests.append(test);
+                });
+
+                if (suite.most_severe_test_result == Test::Result::Fail) {
+                    m_counts.suites_failed++;
+                    file_result.most_severe_test_result = Test::Result::Fail;
+                } else {
+                    if (suite.most_severe_test_result == Test::Result::Skip && file_result.most_severe_test_result == Test::Result::Pass)
+                        file_result.most_severe_test_result = Test::Result::Skip;
+                    m_counts.suites_passed++;
+                }
+
+                file_result.suites.append(suite);
+            });
+
+            m_counts.files_total++;
+
+            file_result.time_taken = get_time_in_ms() - start_time;
+            m_total_elapsed_time_in_ms += file_result.time_taken;
+        },
+        [page_to_load](auto& error, auto) {
+            printf("Failed to load test page: %s (%s)", page_to_load.to_string().characters(), error.characters());
+            cleanup_and_exit();
+        });
+
+    return file_result;
+}
+
+enum Modifier {
+    BG_RED,
+    BG_GREEN,
+    FG_RED,
+    FG_GREEN,
+    FG_ORANGE,
+    FG_GRAY,
+    FG_BLACK,
+    FG_BOLD,
+    ITALIC,
+    CLEAR,
+};
+
+static void print_modifiers(Vector<Modifier> modifiers)
+{
+    for (auto& modifier : modifiers) {
+        auto code = [&]() -> String {
+            switch (modifier) {
+            case BG_RED:
+                return "\033[48;2;255;0;102m";
+            case BG_GREEN:
+                return "\033[48;2;102;255;0m";
+            case FG_RED:
+                return "\033[38;2;255;0;102m";
+            case FG_GREEN:
+                return "\033[38;2;102;255;0m";
+            case FG_ORANGE:
+                return "\033[38;2;255;102;0m";
+            case FG_GRAY:
+                return "\033[38;2;135;139;148m";
+            case FG_BLACK:
+                return "\033[30m";
+            case FG_BOLD:
+                return "\033[1m";
+            case ITALIC:
+                return "\033[3m";
+            case CLEAR:
+                return "\033[0m";
+            }
+            VERIFY_NOT_REACHED();
+        };
+        printf("%s", code().characters());
+    }
+}
+
+void TestRunner::print_file_result(const JSFileResult& file_result) const
+{
+    if (file_result.most_severe_test_result == Test::Result::Fail || file_result.error.has_value()) {
+        print_modifiers({ BG_RED, FG_BLACK, FG_BOLD });
+        printf(" FAIL ");
+        print_modifiers({ CLEAR });
+    } else {
+        if (m_print_times || file_result.most_severe_test_result != Test::Result::Pass) {
+            print_modifiers({ BG_GREEN, FG_BLACK, FG_BOLD });
+            printf(" PASS ");
+            print_modifiers({ CLEAR });
+        } else {
+            return;
+        }
+    }
+
+    printf(" %s", file_result.name.characters());
+
+    if (m_print_times) {
+        print_modifiers({ CLEAR, ITALIC, FG_GRAY });
+        if (file_result.time_taken < 1000) {
+            printf(" (%dms)\n", static_cast<int>(file_result.time_taken));
+        } else {
+            printf(" (%.3fs)\n", file_result.time_taken / 1000.0);
+        }
+        print_modifiers({ CLEAR });
+    } else {
+        printf("\n");
+    }
+
+    if (!file_result.logged_messages.is_empty()) {
+        print_modifiers({ FG_GRAY, FG_BOLD });
+#ifdef __serenity__
+        printf("     ℹ Console output:\n");
+#else
+        // This emoji has a second invisible byte after it. The one above does not
+        printf("    ℹ️  Console output:\n");
+#endif
+        print_modifiers({ CLEAR, FG_GRAY });
+        for (auto& message : file_result.logged_messages)
+            printf("         %s\n", message.characters());
+    }
+
+    if (file_result.error.has_value()) {
+        auto test_error = file_result.error.value();
+
+        print_modifiers({ FG_RED });
+#ifdef __serenity__
+        printf("     ❌ The file failed to parse\n\n");
+#else
+        // No invisible byte here, but the spacing still needs to be altered on the host
+        printf("    ❌ The file failed to parse\n\n");
+#endif
+        print_modifiers({ FG_GRAY });
+        for (auto& message : test_error.hint.split('\n', true)) {
+            printf("         %s\n", message.characters());
+        }
+        print_modifiers({ FG_RED });
+        printf("         %s\n\n", test_error.error.to_string().characters());
+
+        return;
+    }
+
+    if (file_result.most_severe_test_result != Test::Result::Pass) {
+        for (auto& suite : file_result.suites) {
+            if (suite.most_severe_test_result == Test::Result::Pass)
+                continue;
+
+            bool failed = suite.most_severe_test_result == Test::Result::Fail;
+
+            print_modifiers({ FG_GRAY, FG_BOLD });
+
+            if (failed) {
+#ifdef __serenity__
+                printf("     ❌ Suite:  ");
+#else
+                // No invisible byte here, but the spacing still needs to be altered on the host
+                printf("    ❌ Suite:  ");
+#endif
+            } else {
+#ifdef __serenity__
+                printf("     ⚠ Suite:  ");
+#else
+                // This emoji has a second invisible byte after it. The one above does not
+                printf("    ⚠️  Suite:  ");
+#endif
+            }
+
+            print_modifiers({ CLEAR, FG_GRAY });
+
+            if (suite.name == TOP_LEVEL_TEST_NAME) {
+                printf("<top-level>\n");
+            } else {
+                printf("%s\n", suite.name.characters());
+            }
+            print_modifiers({ CLEAR });
+
+            for (auto& test : suite.tests) {
+                if (test.result == Test::Result::Pass)
+                    continue;
+
+                print_modifiers({ FG_GRAY, FG_BOLD });
+                printf("         Test:   ");
+                if (test.result == Test::Result::Fail) {
+                    print_modifiers({ CLEAR, FG_RED });
+                    printf("%s (failed):\n", test.name.characters());
+                    printf("                 %s\n", test.details.characters());
+                } else {
+                    print_modifiers({ CLEAR, FG_ORANGE });
+                    printf("%s (skipped)\n", test.name.characters());
+                }
+                print_modifiers({ CLEAR });
+            }
+        }
+    }
+}
+
+void TestRunner::print_test_results() const
+{
+    printf("\nTest Suites: ");
+    if (m_counts.suites_failed) {
+        print_modifiers({ FG_RED });
+        printf("%d failed, ", m_counts.suites_failed);
+        print_modifiers({ CLEAR });
+    }
+    if (m_counts.suites_passed) {
+        print_modifiers({ FG_GREEN });
+        printf("%d passed, ", m_counts.suites_passed);
+        print_modifiers({ CLEAR });
+    }
+    printf("%d total\n", m_counts.suites_failed + m_counts.suites_passed);
+
+    printf("Tests:       ");
+    if (m_counts.tests_failed) {
+        print_modifiers({ FG_RED });
+        printf("%d failed, ", m_counts.tests_failed);
+        print_modifiers({ CLEAR });
+    }
+    if (m_counts.tests_skipped) {
+        print_modifiers({ FG_ORANGE });
+        printf("%d skipped, ", m_counts.tests_skipped);
+        print_modifiers({ CLEAR });
+    }
+    if (m_counts.tests_passed) {
+        print_modifiers({ FG_GREEN });
+        printf("%d passed, ", m_counts.tests_passed);
+        print_modifiers({ CLEAR });
+    }
+    printf("%d total\n", m_counts.tests_failed + m_counts.tests_passed);
+
+    printf("Files:       %d total\n", m_counts.files_total);
+
+    printf("Time:        ");
+    if (m_total_elapsed_time_in_ms < 1000.0) {
+        printf("%dms\n\n", static_cast<int>(m_total_elapsed_time_in_ms));
+    } else {
+        printf("%-.3fs\n\n", m_total_elapsed_time_in_ms / 1000.0);
+    }
+}
+
+int main(int argc, char** argv)
+{
+    bool print_times = false;
+    bool show_window = false;
+
+#if 0
+    struct sigaction act;
+    memset(&act, 0, sizeof(act));
+    act.sa_flags = SA_NOCLDWAIT;
+    act.sa_handler = handle_sigabrt;
+    int rc = sigaction(SIGABRT, &act, nullptr);
+    if (rc < 0) {
+        perror("sigaction");
+        return 1;
+    }
+#endif
+
+    Core::ArgsParser args_parser;
+    args_parser.add_option(print_times, "Show duration of each test", "show-time", 't');
+    args_parser.add_option(show_window, "Show window while running tests", "window", 'w');
+    args_parser.parse(argc, argv);
+
+    auto app = GUI::Application::construct(argc, argv);
+    auto window = GUI::Window::construct();
+    auto& main_widget = window->set_main_widget<GUI::Widget>();
+    main_widget.set_fill_with_background_color(true);
+    main_widget.set_layout<GUI::VerticalBoxLayout>();
+    auto& view = main_widget.add<Web::InProcessWebView>();
+
+    view.set_document(Web::DOM::Document::create());
+
+    if (show_window) {
+        window->set_title("LibWeb Test Window");
+        window->resize(640, 480);
+        window->show();
+    }
+
+#ifdef __serenity__
+    TestRunner("/home/anon/web-tests", "/home/anon/js-tests", view, print_times).run();
+#else
+    char* serenity_source_dir = getenv("SERENITY_SOURCE_DIR");
+    if (!serenity_source_dir) {
+        printf("test-web requires the SERENITY_SOURCE_DIR environment variable to be set");
+        return 1;
+    }
+    TestRunner(String::formatted("{}/Userland/Libraries/LibWeb/Tests", serenity_source_dir), String::formatted("{}/Userland/Libraries/LibJS/Tests", serenity_source_dir), view, print_times).run();
+#endif
+    return 0;
+}