/* * Copyright (c) 2020, Matthew Olsson * Copyright (c) 2020-2021, Linus Groh * Copyright (c) 2021, Ali Mohammad Pur * Copyright (c) 2021, Andreas Kling * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __serenity__ # include #endif #define STRCAT(x, y) __STRCAT(x, y) #define STRSTRCAT(x, y) __STRSTRCAT(x, y) #define __STRCAT(x, y) x #y #define __STRSTRCAT(x, y) x y // Note: This is a little weird, so here's an explanation: // If the vararg isn't given, the tuple initializer will simply expand to `fn, ::Test::JS::__testjs_last<1>()` // and if it _is_ given (say as `A`), the tuple initializer will expand to `fn, ::Test::JS::__testjs_last<1, A>()`, which will end up being evaluated as `A` // and if multiple args are given, the static_assert will be sad. #define __TESTJS_REGISTER_GLOBAL_FUNCTION(name, fn, ...) \ struct __TestJS_register_##fn { \ static_assert( \ ::Test::JS::__testjs_count(__VA_ARGS__) <= 1, \ STRCAT(STRSTRCAT(STRCAT("Expected at most three arguments to TESTJS_GLOBAL_FUNCTION at line", __LINE__), ", in file "), __FILE__)); \ __TestJS_register_##fn() noexcept \ { \ ::Test::JS::s_exposed_global_functions.set( \ name, \ { fn, ::Test::JS::__testjs_last<1, ##__VA_ARGS__>() }); \ } \ } __testjs_register_##fn {}; #define TESTJS_GLOBAL_FUNCTION(function, exposed_name, ...) \ JS_DECLARE_NATIVE_FUNCTION(function); \ __TESTJS_REGISTER_GLOBAL_FUNCTION(#exposed_name, function, ##__VA_ARGS__); \ JS_DEFINE_NATIVE_FUNCTION(function) #define TESTJS_MAIN_HOOK() \ struct __TestJS_main_hook { \ __TestJS_main_hook() \ { \ ::Test::JS::g_main_hook = hook; \ } \ static void hook(); \ } __testjs_common_register_##name {}; \ void __TestJS_main_hook::hook() #define TESTJS_PROGRAM_FLAG(flag, help_string, long_name, short_name) \ bool flag { false }; \ struct __TestJS_flag_hook_##flag { \ __TestJS_flag_hook_##flag() \ { \ ::Test::JS::g_extra_args.set(&(flag), { help_string, long_name, short_name }); \ }; \ } __testjs_flag_hook_##flag; #define TEST_ROOT(path) \ String Test::JS::g_test_root_fragment = path #define TESTJS_RUN_FILE_FUNCTION(...) \ struct __TestJS_run_file { \ __TestJS_run_file() \ { \ ::Test::JS::g_run_file = hook; \ } \ static ::Test::JS::IntermediateRunFileResult hook(const String&, JS::Interpreter&); \ } __testjs_common_run_file {}; \ ::Test::JS::IntermediateRunFileResult __TestJS_run_file::hook(__VA_ARGS__) namespace Test::JS { namespace JS = ::JS; template static consteval size_t __testjs_count(Args...) { return sizeof...(Args); } template static consteval size_t __testjs_last() { return (AK::Detail::IntegralConstant {}, ...).value; } static constexpr auto TOP_LEVEL_TEST_NAME = "__$$TOP_LEVEL$$__"; extern RefPtr g_vm; extern bool g_collect_on_every_allocation; #ifdef JS_TRACK_ZOMBIE_CELLS extern bool g_zombify_dead_cells; #endif extern bool g_run_bytecode; extern String g_currently_running_test; struct FunctionWithLength { JS::ThrowCompletionOr (*function)(JS::VM&, JS::GlobalObject&); size_t length { 0 }; }; extern HashMap s_exposed_global_functions; extern String g_test_root_fragment; extern String g_test_root; extern int g_test_argc; extern char** g_test_argv; extern Function g_main_hook; extern HashMap> g_extra_args; struct ParserError { JS::Parser::Error error; String hint; }; struct JSFileResult { String name; Optional error {}; double time_taken { 0 }; // A failed test takes precedence over a skipped test, which both have // precedence over a passed test Test::Result most_severe_test_result { Test::Result::Pass }; Vector suites {}; Vector logged_messages {}; }; enum class RunFileHookResult { RunAsNormal, SkipFile, }; using IntermediateRunFileResult = AK::Result; extern IntermediateRunFileResult (*g_run_file)(const String&, JS::Interpreter&); class TestRunner : public ::Test::TestRunner { public: TestRunner(String test_root, String common_path, bool print_times, bool print_progress, bool print_json) : ::Test::TestRunner(move(test_root), print_times, print_progress, print_json) , m_common_path(move(common_path)) { g_test_root = m_test_root; } virtual ~TestRunner() = default; protected: virtual void do_run_single_test(const String& test_path, size_t, size_t) override; virtual Vector get_test_paths() const override; virtual JSFileResult run_file_test(const String& test_path); void print_file_result(const JSFileResult& file_result) const; String m_common_path; RefPtr m_test_script; }; class TestRunnerGlobalObject final : public JS::GlobalObject { JS_OBJECT(TestRunnerGlobalObject, JS::GlobalObject); public: TestRunnerGlobalObject() = default; virtual ~TestRunnerGlobalObject() override = default; virtual void initialize_global_object() override; }; inline void TestRunnerGlobalObject::initialize_global_object() { Base::initialize_global_object(); define_direct_property("global", this, JS::Attribute::Enumerable); for (auto& entry : s_exposed_global_functions) { define_native_function( entry.key, [fn = entry.value.function](auto& vm, auto& global_object) { return fn(vm, global_object); }, entry.value.length, JS::default_attributes); } } inline ByteBuffer load_entire_file(StringView path) { auto file_or_error = Core::File::open(path, Core::OpenMode::ReadOnly); if (file_or_error.is_error()) { warnln("Failed to open the following file: \"{}\"", path); cleanup_and_exit(); } auto file = file_or_error.release_value(); return file->read_all(); } inline AK::Result, ParserError> parse_script(StringView path, JS::Realm& realm) { auto contents = load_entire_file(path); auto script_or_errors = JS::Script::parse(contents, realm, path); if (script_or_errors.is_error()) { auto errors = script_or_errors.release_error(); return ParserError { errors[0], errors[0].source_location_hint(contents) }; } return script_or_errors.release_value(); } inline AK::Result, ParserError> parse_module(StringView path, JS::Realm& realm) { auto contents = load_entire_file(path); auto script_or_errors = JS::SourceTextModule::parse(contents, realm, path); if (script_or_errors.is_error()) { auto errors = script_or_errors.release_error(); return ParserError { errors[0], errors[0].source_location_hint(contents) }; } return script_or_errors.release_value(); } inline Optional get_test_results(JS::Interpreter& interpreter) { auto results = MUST(interpreter.global_object().get("__TestResults__")); auto json_string = TRY_OR_DISCARD(JS::JSONObject::stringify_impl(interpreter.global_object(), results, JS::js_undefined(), JS::js_undefined())); auto json = JsonValue::from_string(json_string); if (!json.has_value()) return {}; return json.value(); } inline void TestRunner::do_run_single_test(const String& test_path, size_t, size_t) { auto file_result = run_file_test(test_path); if (!m_print_json) print_file_result(file_result); } inline Vector TestRunner::get_test_paths() const { Vector paths; iterate_directory_recursively(m_test_root, [&](const String& file_path) { if (!file_path.ends_with(".js")) return; if (!file_path.ends_with("test-common.js")) paths.append(file_path); }); quick_sort(paths); return paths; } inline JSFileResult TestRunner::run_file_test(const String& test_path) { g_currently_running_test = test_path; #ifdef __serenity__ auto string_id = perf_register_string(test_path.characters(), test_path.length()); perf_event(PERF_EVENT_SIGNPOST, string_id, 0); #endif double start_time = get_time_in_ms(); auto interpreter = JS::Interpreter::create(*g_vm); // FIXME: This is a hack while we're refactoring Interpreter/VM stuff. JS::VM::InterpreterExecutionScope scope(*interpreter); interpreter->heap().set_should_collect_on_every_allocation(g_collect_on_every_allocation); #ifdef JS_TRACK_ZOMBIE_CELLS interpreter->heap().set_zombify_dead_cells(g_zombify_dead_cells); #endif if (g_run_file) { auto result = g_run_file(test_path, *interpreter); if (result.is_error() && result.error() == RunFileHookResult::SkipFile) { return { test_path, {}, 0, Test::Result::Skip, {}, {} }; } if (!result.is_error()) { auto value = result.release_value(); for (auto& suite : value.suites) { if (suite.most_severe_test_result == Result::Pass) m_counts.suites_passed++; else if (suite.most_severe_test_result == Result::Fail) m_counts.suites_failed++; for (auto& test : suite.tests) { if (test.result == Result::Pass) m_counts.tests_passed++; else if (test.result == Result::Fail) m_counts.tests_failed++; else if (test.result == Result::Skip) m_counts.tests_skipped++; } } ++m_counts.files_total; m_total_elapsed_time_in_ms += value.time_taken; return value; } } if (!m_test_script) { auto result = parse_script(m_common_path, interpreter->realm()); if (result.is_error()) { warnln("Unable to parse test-common.js"); warnln("{}", result.error().error.to_string()); warnln("{}", result.error().hint); cleanup_and_exit(); } m_test_script = result.release_value(); } if (g_run_bytecode) { auto executable = JS::Bytecode::Generator::generate(m_test_script->parse_node()); executable.name = test_path; if (JS::Bytecode::g_dump_bytecode) executable.dump(); JS::Bytecode::Interpreter bytecode_interpreter(interpreter->global_object(), interpreter->realm()); bytecode_interpreter.run(executable); } else { interpreter->run(interpreter->global_object(), m_test_script->parse_node()); } VERIFY(!g_vm->exception()); auto file_script = parse_script(test_path, interpreter->realm()); if (file_script.is_error()) return { test_path, file_script.error() }; if (g_run_bytecode) { auto executable = JS::Bytecode::Generator::generate(file_script.value()->parse_node()); executable.name = test_path; if (JS::Bytecode::g_dump_bytecode) executable.dump(); JS::Bytecode::Interpreter bytecode_interpreter(interpreter->global_object(), interpreter->realm()); bytecode_interpreter.run(executable); } else { interpreter->run(interpreter->global_object(), file_script.value()->parse_node()); } if (g_vm->exception()) g_vm->clear_exception(); auto test_json = get_test_results(*interpreter); if (!test_json.has_value()) { warnln("Received malformed JSON from test \"{}\"", test_path); cleanup_and_exit(); } JSFileResult file_result { test_path.substring(m_test_root.length() + 1, test_path.length() - m_test_root.length() - 1) }; // Collect logged messages auto user_output = MUST(interpreter->global_object().get("__UserOutput__")); auto& arr = user_output.as_array(); for (auto& entry : arr.indexed_properties()) { auto message = MUST(arr.get(entry.index())); file_result.logged_messages.append(message.to_string_without_side_effects()); } test_json.value().as_object().for_each_member([&](const String& suite_name, const JsonValue& suite_value) { Test::Suite suite { suite_name }; VERIFY(suite_value.is_object()); suite_value.as_object().for_each_member([&](const String& test_name, const JsonValue& test_value) { Test::Case test { test_name, Test::Result::Fail, "" }; VERIFY(test_value.is_object()); VERIFY(test_value.as_object().has("result")); auto result = test_value.as_object().get("result"); VERIFY(result.is_string()); auto result_string = result.as_string(); if (result_string == "pass") { test.result = Test::Result::Pass; m_counts.tests_passed++; } else if (result_string == "fail") { test.result = Test::Result::Fail; m_counts.tests_failed++; suite.most_severe_test_result = Test::Result::Fail; VERIFY(test_value.as_object().has("details")); auto details = test_value.as_object().get("details"); VERIFY(result.is_string()); test.details = details.as_string(); } else { test.result = Test::Result::Skip; if (suite.most_severe_test_result == Test::Result::Pass) suite.most_severe_test_result = Test::Result::Skip; m_counts.tests_skipped++; } suite.tests.append(test); }); if (suite.most_severe_test_result == Test::Result::Fail) { m_counts.suites_failed++; file_result.most_severe_test_result = Test::Result::Fail; } else { if (suite.most_severe_test_result == Test::Result::Skip && file_result.most_severe_test_result == Test::Result::Pass) file_result.most_severe_test_result = Test::Result::Skip; m_counts.suites_passed++; } file_result.suites.append(suite); }); m_counts.files_total++; file_result.time_taken = get_time_in_ms() - start_time; m_total_elapsed_time_in_ms += file_result.time_taken; return file_result; } inline void TestRunner::print_file_result(const JSFileResult& file_result) const { if (file_result.most_severe_test_result == Test::Result::Fail || file_result.error.has_value()) { print_modifiers({ BG_RED, FG_BLACK, FG_BOLD }); out(" FAIL "); print_modifiers({ CLEAR }); } else { if (m_print_times || file_result.most_severe_test_result != Test::Result::Pass) { print_modifiers({ BG_GREEN, FG_BLACK, FG_BOLD }); out(" PASS "); print_modifiers({ CLEAR }); } else { return; } } out(" {}", file_result.name); if (m_print_times) { print_modifiers({ CLEAR, ITALIC, FG_GRAY }); if (file_result.time_taken < 1000) { outln(" ({}ms)", static_cast(file_result.time_taken)); } else { outln(" ({:3}s)", file_result.time_taken / 1000.0); } print_modifiers({ CLEAR }); } else { outln(); } if (!file_result.logged_messages.is_empty()) { print_modifiers({ FG_GRAY, FG_BOLD }); #ifdef __serenity__ outln(" ℹ Console output:"); #else // This emoji has a second invisible byte after it. The one above does not outln(" ℹ️ Console output:"); #endif print_modifiers({ CLEAR, FG_GRAY }); for (auto& message : file_result.logged_messages) outln(" {}", message); } if (file_result.error.has_value()) { auto test_error = file_result.error.value(); print_modifiers({ FG_RED }); #ifdef __serenity__ outln(" ❌ The file failed to parse"); #else // No invisible byte here, but the spacing still needs to be altered on the host outln(" ❌ The file failed to parse"); #endif outln(); print_modifiers({ FG_GRAY }); for (auto& message : test_error.hint.split('\n', true)) { outln(" {}", message); } print_modifiers({ FG_RED }); outln(" {}", test_error.error.to_string()); outln(); return; } if (file_result.most_severe_test_result != Test::Result::Pass) { for (auto& suite : file_result.suites) { if (suite.most_severe_test_result == Test::Result::Pass) continue; bool failed = suite.most_severe_test_result == Test::Result::Fail; print_modifiers({ FG_GRAY, FG_BOLD }); if (failed) { #ifdef __serenity__ out(" ❌ Suite: "); #else // No invisible byte here, but the spacing still needs to be altered on the host out(" ❌ Suite: "); #endif } else { #ifdef __serenity__ out(" ⚠ Suite: "); #else // This emoji has a second invisible byte after it. The one above does not out(" ⚠️ Suite: "); #endif } print_modifiers({ CLEAR, FG_GRAY }); if (suite.name == TOP_LEVEL_TEST_NAME) { outln(""); } else { outln("{}", suite.name); } print_modifiers({ CLEAR }); for (auto& test : suite.tests) { if (test.result == Test::Result::Pass) continue; print_modifiers({ FG_GRAY, FG_BOLD }); out(" Test: "); if (test.result == Test::Result::Fail) { print_modifiers({ CLEAR, FG_RED }); outln("{} (failed):", test.name); outln(" {}", test.details); } else { print_modifiers({ CLEAR, FG_ORANGE }); outln("{} (skipped)", test.name); } print_modifiers({ CLEAR }); } } } } }