path: root/Userland/Utilities/run-tests.cpp

/*
 * Copyright (c) 2021, Andrew Kaster <akaster@serenityos.org>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 */

#include <AK/LexicalPath.h>
#include <LibCore/ArgsParser.h>
#include <LibCore/ConfigFile.h>
#include <LibCore/File.h>
#include <LibCore/System.h>
#include <LibCoredump/Backtrace.h>
#include <LibMain/Main.h>
#include <LibRegex/Regex.h>
#include <LibTest/TestRunner.h>
#include <signal.h>
#include <spawn.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <unistd.h>

namespace Test {
TestRunner* TestRunner::s_the = nullptr;
}
using Test::get_time_in_ms;
using Test::print_modifiers;

struct FileResult {
    LexicalPath file_path;
    double time_taken { 0 };
    Test::Result result { Test::Result::Pass };
    int stdout_err_fd { -1 };
    pid_t child_pid { 0 };
};

String g_currently_running_test;

class TestRunner : public ::Test::TestRunner {
public:
    TestRunner(String test_root, Regex<PosixExtended> exclude_regex, NonnullRefPtr<Core::ConfigFile> config, Regex<PosixExtended> skip_regex, bool run_skipped_tests, bool print_progress, bool print_json, bool print_all_output, bool print_times = true)
        : ::Test::TestRunner(move(test_root), print_times, print_progress, print_json)
        , m_exclude_regex(move(exclude_regex))
        , m_config(move(config))
        , m_skip_regex(move(skip_regex))
        , m_run_skipped_tests(run_skipped_tests)
        , m_print_all_output(print_all_output)
    {
        if (!run_skipped_tests) {
            m_skip_directories = m_config->read_entry("Global", "SkipDirectories", "").split(' ');
            m_skip_files = m_config->read_entry("Global", "SkipTests", "").split(' ');
        }
    }

    virtual ~TestRunner() = default;

protected:
    virtual void do_run_single_test(String const& test_path, size_t current_text_index, size_t num_tests) override;
    virtual Vector<String> get_test_paths() const override;
    virtual Vector<String> const* get_failed_test_names() const override { return &m_failed_test_names; }

    virtual FileResult run_test_file(String const& test_path);

    bool should_skip_test(LexicalPath const& test_path);

    Regex<PosixExtended> m_exclude_regex;
    NonnullRefPtr<Core::ConfigFile> m_config;
    Vector<String> m_skip_directories;
    Vector<String> m_skip_files;
    Vector<String> m_failed_test_names;
    Regex<PosixExtended> m_skip_regex;
    bool m_run_skipped_tests { false };
    bool m_print_all_output { false };
};

Vector<String> TestRunner::get_test_paths() const
{
    Vector<String> paths;
    Test::iterate_directory_recursively(m_test_root, [&](String const& file_path) {
        if (access(file_path.characters(), R_OK | X_OK) != 0)
            return;
        auto result = m_exclude_regex.match(file_path, PosixFlags::Global);
        if (!result.success) // must NOT match the regex to be a valid test file
            paths.append(file_path);
    });
    quick_sort(paths);
    return paths;
}

bool TestRunner::should_skip_test(LexicalPath const& test_path)
{
    if (m_run_skipped_tests)
        return false;

    for (String const& dir : m_skip_directories) {
        if (test_path.dirname().contains(dir))
            return true;
    }
    for (String const& file : m_skip_files) {
        if (test_path.basename().contains(file))
            return true;
    }
    auto result = m_skip_regex.match(test_path.basename(), PosixFlags::Global);
    if (result.success)
        return true;

    return false;
}

void TestRunner::do_run_single_test(String const& test_path, size_t current_test_index, size_t num_tests)
{
    g_currently_running_test = test_path;
    auto test_relative_path = LexicalPath::relative_path(test_path, m_test_root);
    outln(" START  {} ({}/{})", test_relative_path, current_test_index, num_tests);
    fflush(stdout); // we really want to see the start text in case the test hangs
    auto test_result = run_test_file(test_path);

    switch (test_result.result) {
    case Test::Result::Pass:
        ++m_counts.tests_passed;
        break;
    case Test::Result::Skip:
        ++m_counts.tests_skipped;
        break;
    case Test::Result::Fail:
        ++m_counts.tests_failed;
        break;
    case Test::Result::Crashed:
        ++m_counts.tests_failed; // FIXME: tests_crashed
        break;
    }
    if (test_result.result != Test::Result::Skip)
        ++m_counts.files_total;

    m_total_elapsed_time_in_ms += test_result.time_taken;

    bool crashed_or_failed = test_result.result == Test::Result::Fail || test_result.result == Test::Result::Crashed;
    bool print_stdout_stderr = crashed_or_failed || m_print_all_output;
    if (crashed_or_failed) {
        m_failed_test_names.append(test_path);
        print_modifiers({ Test::BG_RED, Test::FG_BLACK, Test::FG_BOLD });
        out("{}", test_result.result == Test::Result::Fail ? " FAIL  " : "CRASHED");
        print_modifiers({ Test::CLEAR });
        if (test_result.result == Test::Result::Crashed) {
            auto pid_search_string = String::formatted("_{}_", test_result.child_pid);
            Core::DirIterator iterator("/tmp/coredump"sv);
            if (!iterator.has_error()) {
                while (iterator.has_next()) {
                    auto path = iterator.next_full_path();
                    if (!path.contains(pid_search_string))
                        continue;

                    auto reader = Coredump::Reader::create(path);
                    if (!reader)
                        break;

                    dbgln("Last crash backtrace for {} (was pid {}):", test_path, test_result.child_pid);
                    reader->for_each_thread_info([&](auto thread_info) {
                        Coredump::Backtrace thread_backtrace(*reader, thread_info);
                        auto tid = thread_info.tid; // Note: Yoinking this out of the struct because we can't pass a reference to it (as it's a misaligned field in a packed struct)
                        dbgln("Thread {}", tid);
                        for (auto const& entry : thread_backtrace.entries())
                            dbgln("- {}", entry.to_string(true));
                        return IterationDecision::Continue;
                    });
                    break;
                }
            }
        }
    } else {
        print_modifiers({ Test::BG_GREEN, Test::FG_BLACK, Test::FG_BOLD });
        out(" PASS  ");
        print_modifiers({ Test::CLEAR });
    }

    out(" {}", test_relative_path);

    print_modifiers({ Test::CLEAR, Test::ITALIC, Test::FG_GRAY });
    if (test_result.time_taken < 1000) {
        outln(" ({}ms)", static_cast<int>(test_result.time_taken));
    } else {
        outln(" ({:3}s)", test_result.time_taken / 1000.0);
    }
    print_modifiers({ Test::CLEAR });

    if (test_result.result != Test::Result::Pass) {
        print_modifiers({ Test::FG_GRAY, Test::FG_BOLD });
        out("         Test:   ");
        if (crashed_or_failed) {
            print_modifiers({ Test::CLEAR, Test::FG_RED });
            outln("{} ({})", test_result.file_path.basename(), test_result.result == Test::Result::Fail ? "failed" : "crashed");
        } else {
            print_modifiers({ Test::CLEAR, Test::FG_ORANGE });
            outln("{} (skipped)", test_result.file_path.basename());
        }
        print_modifiers({ Test::CLEAR });
    }

    // Make sure our clear modifiers goes through before we dump file output via write(2)
    fflush(stdout);

    if (print_stdout_stderr && test_result.stdout_err_fd > 0) {
        int ret = lseek(test_result.stdout_err_fd, 0, SEEK_SET);
        VERIFY(ret == 0);
        for (;;) {
            char buf[32768];
            ssize_t nread = read(test_result.stdout_err_fd, buf, sizeof(buf));
            if (nread == 0)
                break;
            if (nread < 0) {
                perror("read");
                break;
            }
            size_t already_written = 0;
            while (already_written < (size_t)nread) {
                ssize_t nwritten = write(STDOUT_FILENO, buf + already_written, nread - already_written);
                if (nwritten < 0) {
                    perror("write");
                    break;
                }
                already_written += nwritten;
            }
        }
    }

    close(test_result.stdout_err_fd);
}

FileResult TestRunner::run_test_file(String const& test_path)
{
    double start_time = get_time_in_ms();

    auto path_for_test = LexicalPath(test_path);
    if (should_skip_test(path_for_test)) {
        return FileResult { move(path_for_test), 0.0, Test::Result::Skip, -1 };
    }

    // FIXME: actual error handling, mark test as :yaksplode: if any are bad instead of VERIFY
    posix_spawn_file_actions_t file_actions;
    posix_spawn_file_actions_init(&file_actions);
    char child_out_err_path[] = "/tmp/run-tests.XXXXXX";
    int child_out_err_file = mkstemp(child_out_err_path);
    VERIFY(child_out_err_file >= 0);

    String dirname = path_for_test.dirname();
    String basename = path_for_test.basename();

    (void)posix_spawn_file_actions_adddup2(&file_actions, child_out_err_file, STDOUT_FILENO);
    (void)posix_spawn_file_actions_adddup2(&file_actions, child_out_err_file, STDERR_FILENO);
    (void)posix_spawn_file_actions_addchdir(&file_actions, dirname.characters());

    Vector<char const*, 4> argv;
    argv.append(basename.characters());
    auto extra_args = m_config->read_entry(path_for_test.basename(), "Arguments", "").split(' ');
    for (auto& arg : extra_args)
        argv.append(arg.characters());
    argv.append(nullptr);

    pid_t child_pid = -1;
    // FIXME: Do we really want to copy test runner's entire env?
    int ret = posix_spawn(&child_pid, test_path.characters(), &file_actions, nullptr, const_cast<char* const*>(argv.data()), environ);
    VERIFY(ret == 0);
    VERIFY(child_pid > 0);

    int wstatus;

    Test::Result test_result = Test::Result::Fail;
    for (size_t num_waits = 0; num_waits < 2; ++num_waits) {
        ret = waitpid(child_pid, &wstatus, 0); // intentionally not setting WCONTINUED
        if (ret != child_pid)
            break; // we'll end up with a failure

        if (WIFEXITED(wstatus)) {
            if (wstatus == 0) {
                test_result = Test::Result::Pass;
            }
            break;
        } else if (WIFSIGNALED(wstatus)) {
            test_result = Test::Result::Crashed;
            break;
        } else if (WIFSTOPPED(wstatus)) {
            outln("{} was stopped unexpectedly, sending SIGCONT", test_path);
            kill(child_pid, SIGCONT);
        }
    }

    // Remove the child's stdout from /tmp. This does cause the temp file to be observable
    // while the test is executing, but if it hangs that might even be a bonus :)
    ret = unlink(child_out_err_path);
    VERIFY(ret == 0);

    return FileResult { move(path_for_test), get_time_in_ms() - start_time, test_result, child_out_err_file, child_pid };
}

ErrorOr<int> serenity_main(Main::Arguments arguments)
{

    auto program_name = LexicalPath::basename(arguments.strings[0]);

#ifdef SIGINFO
    TRY(Core::System::signal(SIGINFO, [](int) {
        static char buffer[4096];
        auto& counts = ::Test::TestRunner::the()->counts();
        int len = snprintf(buffer, sizeof(buffer), "Pass: %d, Fail: %d, Skip: %d\nCurrent test: %s\n", counts.tests_passed, counts.tests_failed, counts.tests_skipped, g_currently_running_test.characters());
        write(STDOUT_FILENO, buffer, len);
    }));
#endif

    bool print_progress =
#ifdef __serenity__
        true; // Use OSC 9 to print progress
#else
        false;
#endif
    bool print_json = false;
    bool print_all_output = false;
    bool run_benchmarks = false;
    bool run_skipped_tests = false;
    char const* specified_test_root = nullptr;
    String test_glob;
    String exclude_pattern;
    String config_file;

    Core::ArgsParser args_parser;
    args_parser.add_option(Core::ArgsParser::Option {
        .argument_mode = Core::ArgsParser::OptionArgumentMode::Required,
        .help_string = "Show progress with OSC 9 (true, false)",
        .long_name = "show-progress",
        .short_name = 'p',
        .accept_value = [&](auto* str) {
            if ("true"sv == str)
                print_progress = true;
            else if ("false"sv == str)
                print_progress = false;
            else
                return false;
            return true;
        },
    });
    args_parser.add_option(print_json, "Show results as JSON", "json", 'j');
    args_parser.add_option(print_all_output, "Show all test output", "verbose", 'v');
    args_parser.add_option(run_benchmarks, "Run benchmarks as well", "benchmarks", 'b');
    args_parser.add_option(run_skipped_tests, "Run all matching tests, even those marked as 'skip'", "all", 'a');
    args_parser.add_option(test_glob, "Only run tests matching the given glob", "filter", 'f', "glob");
    args_parser.add_option(exclude_pattern, "Regular expression to use to exclude paths from being considered tests", "exclude-pattern", 'e', "pattern");
    args_parser.add_option(config_file, "Configuration file to use", "config-file", 'c', "filename");
    args_parser.add_positional_argument(specified_test_root, "Tests root directory", "path", Core::ArgsParser::Required::No);
    args_parser.parse(arguments);

    test_glob = String::formatted("*{}*", test_glob);

    if (getenv("DISABLE_DBG_OUTPUT")) {
        AK::set_debug_enabled(false);
    }

    // Make UBSAN deadly for all tests we run by default.
    TRY(Core::System::setenv("UBSAN_OPTIONS"sv, "halt_on_error=1"sv, true));

    if (!run_benchmarks)
        TRY(Core::System::setenv("TESTS_ONLY"sv, "1"sv, true));

    String test_root;

    if (specified_test_root) {
        test_root = String { specified_test_root };
    } else {
        test_root = "/usr/Tests";
    }
    if (!Core::File::is_directory(test_root)) {
        warnln("Test root is not a directory: {}", test_root);
        return 1;
    }

    test_root = Core::File::real_path_for(test_root);

    auto void_or_error = Core::System::chdir(test_root);
    if (void_or_error.is_error()) {
        warnln("chdir failed: {}", void_or_error.error());
        return void_or_error.release_error();
    }

    auto config_or_error = config_file.is_empty() ? Core::ConfigFile::open_for_app("Tests") : Core::ConfigFile::open(config_file);
    if (config_or_error.is_error()) {
        warnln("Failed to open configuration file ({}): {}", config_file.is_empty() ? "User config for Tests" : config_file.characters(), config_or_error.error());
        return config_or_error.release_error();
    }
    auto config = config_or_error.release_value();

    if (config->num_groups() == 0)
        warnln("Empty configuration file ({}) loaded!", config_file.is_empty() ? "User config for Tests" : config_file.characters());

    if (exclude_pattern.is_empty())
        exclude_pattern = config->read_entry("Global", "NotTestsPattern", "$^"); // default is match nothing (aka match end then beginning)

    Regex<PosixExtended> exclude_regex(exclude_pattern, {});
    if (exclude_regex.parser_result.error != regex::Error::NoError) {
        warnln("Exclude pattern \"{}\" is invalid", exclude_pattern);
        return 1;
    }

    // we need to preconfigure this, because we can't autoinitialize Regex types
    // in the Testrunner
    auto skip_regex_pattern = config->read_entry("Global", "SkipRegex", "$^");
    Regex<PosixExtended> skip_regex { skip_regex_pattern, {} };
    if (skip_regex.parser_result.error != regex::Error::NoError) {
        warnln("SkipRegex pattern \"{}\" is invalid", skip_regex_pattern);
        return 1;
    }

    TestRunner test_runner(test_root, move(exclude_regex), move(config), move(skip_regex), run_skipped_tests, print_progress, print_json, print_all_output);
    test_runner.run(test_glob);

    return test_runner.counts().tests_failed;
}