AK+Userland: Move AK/TestSuite.h into LibTest and rework Tests' CMake

As many macros as possible are moved to Macros.h, while the
macros to create a test case are moved to TestCase.h. TestCase is now
the only user-facing header for creating a test case. TestSuite and its
helpers have moved into a .cpp file. Instead of requiring a TEST_MAIN
macro to be instantiated into the test file, a TestMain.cpp file is
provided instead that will be linked against each test. This has the
side effect that, if we wanted to have test cases split across multiple
files, it's as simple as adding them all to the same executable.

The test main should be portable to kernel mode as well, so if
there's a set of tests that should be run in self-test mode in kernel
space, we can accomodate that.

A new serenity_test CMake function streamlines adding a new test with
arguments for the test source file, subdirectory under /usr/Tests to
install the test application and an optional list of libraries to link
against the test application. To accomodate future test where the
provided TestMain.cpp is not suitable (e.g. test-js), a CUSTOM_MAIN
parameter can be passed to the function to not link against the
boilerplate main function.

6 files changed, 392 insertions, 0 deletions

diff --git a/Userland/Libraries/LibTest/CMakeLists.txt b/Userland/Libraries/LibTest/CMakeLists.txt
index a924ea3e3f..7816fdac71 100644
--- a/Userland/Libraries/LibTest/CMakeLists.txt
+++ b/Userland/Libraries/LibTest/CMakeLists.txt
@@ -1 +1,8 @@
 serenity_install_sources("Userland/Libraries/LibTest")
+
+set(SOURCES
+    TestSuite.cpp
+)
+
+serenity_lib(LibTest test)
+target_link_libraries(LibTest LibC)
diff --git a/Userland/Libraries/LibTest/Macros.h b/Userland/Libraries/LibTest/Macros.h
new file mode 100644
index 0000000000..b4704508e2
--- /dev/null
+++ b/Userland/Libraries/LibTest/Macros.h
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2021, Andrew Kaster <akaster@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Assertions.h>
+#include <AK/CheckedFormatString.h>
+
+namespace AK {
+template<typename... Parameters>
+void warnln(CheckedFormatString<Parameters...>&& fmtstr, const Parameters&...);
+}
+
+namespace Test {
+// Declare a helper so that we can call it from VERIFY in included headers
+void current_test_case_did_fail();
+}
+
+#undef VERIFY
+#define VERIFY(x)                                                                                    \
+    do {                                                                                             \
+        if (!(x)) {                                                                                  \
+            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: VERIFY({}) failed", __FILE__, __LINE__, #x); \
+            ::Test::current_test_case_did_fail();                                                    \
+        }                                                                                            \
+    } while (false)
+
+#undef VERIFY_NOT_REACHED
+#define VERIFY_NOT_REACHED()                                                                           \
+    do {                                                                                               \
+        ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: VERIFY_NOT_REACHED() called", __FILE__, __LINE__); \
+        ::abort();                                                                                     \
+    } while (false)
+
+#undef TODO
+#define TODO()                                                                                   \
+    do {                                                                                         \
+        ::AK::warnln(stderr, "\033[31;1mFAIL\033[0m: {}:{}: TODO() called", __FILE__, __LINE__); \
+        ::abort();                                                                               \
+    } while (false)
+
+#define EXPECT_EQ(a, b)                                                                                                                                                                      \
+    do {                                                                                                                                                                                     \
+        auto lhs = (a);                                                                                                                                                                      \
+        auto rhs = (b);                                                                                                                                                                      \
+        if (lhs != rhs) {                                                                                                                                                                    \
+            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
+            ::Test::current_test_case_did_fail();                                                                                                                                            \
+        }                                                                                                                                                                                    \
+    } while (false)
+
+// If you're stuck and `EXPECT_EQ` seems to refuse to print anything useful,
+// try this: It'll spit out a nice compiler error telling you why it doesn't print.
+#define EXPECT_EQ_FORCE(a, b)                                                                                                                    \
+    do {                                                                                                                                         \
+        auto lhs = (a);                                                                                                                          \
+        auto rhs = (b);                                                                                                                          \
+        if (lhs != rhs) {                                                                                                                        \
+            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \
+            ::Test::current_test_case_did_fail();                                                                                                \
+        }                                                                                                                                        \
+    } while (false)
+
+#define EXPECT(x)                                                                                    \
+    do {                                                                                             \
+        if (!(x)) {                                                                                  \
+            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \
+            ::Test::current_test_case_did_fail();                                                    \
+        }                                                                                            \
+    } while (false)
+
+#define EXPECT_APPROXIMATE(a, b)                                                                                \
+    do {                                                                                                        \
+        auto expect_close_lhs = a;                                                                              \
+        auto expect_close_rhs = b;                                                                              \
+        auto expect_close_diff = static_cast<double>(expect_close_lhs) - static_cast<double>(expect_close_rhs); \
+        if (fabs(expect_close_diff) > 0.0000005) {                                                              \
+            ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})"                             \
+                         " failed with lhs={}, rhs={}, (lhs-rhs)={}",                                           \
+                __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff);             \
+            ::Test::current_test_case_did_fail();                                                               \
+        }                                                                                                       \
+    } while (false)
diff --git a/Userland/Libraries/LibTest/TestCase.h b/Userland/Libraries/LibTest/TestCase.h
new file mode 100644
index 0000000000..d0fb713567
--- /dev/null
+++ b/Userland/Libraries/LibTest/TestCase.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2021, Andrew Kaster <akaster@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibTest/Macros.h> // intentionally first -- we redefine VERIFY and friends in here
+
+#include <AK/Function.h>
+#include <AK/NonnullRefPtr.h>
+#include <AK/RefCounted.h>
+#include <AK/String.h>
+
+namespace Test {
+
+using TestFunction = Function<void()>;
+
+class TestCase : public RefCounted<TestCase> {
+public:
+    TestCase(const String& name, TestFunction&& fn, bool is_benchmark)
+        : m_name(name)
+        , m_function(move(fn))
+        , m_is_benchmark(is_benchmark)
+    {
+    }
+
+    bool is_benchmark() const { return m_is_benchmark; }
+    const String& name() const { return m_name; }
+    const TestFunction& func() const { return m_function; }
+
+private:
+    String m_name;
+    TestFunction m_function;
+    bool m_is_benchmark;
+};
+
+// Helper to hide implementation of TestSuite from users
+void add_test_case_to_suite(const NonnullRefPtr<TestCase>& test_case);
+
+}
+
+#define __TESTCASE_FUNC(x) __test_##x
+#define __TESTCASE_TYPE(x) __TestCase_##x
+
+#define TEST_CASE(x)                                                                                    \
+    static void __TESTCASE_FUNC(x)();                                                                   \
+    struct __TESTCASE_TYPE(x) {                                                                         \
+        __TESTCASE_TYPE(x)                                                                              \
+        () { add_test_case_to_suite(adopt_ref(*new ::Test::TestCase(#x, __TESTCASE_FUNC(x), false))); } \
+    };                                                                                                  \
+    static struct __TESTCASE_TYPE(x) __TESTCASE_TYPE(x);                                                \
+    static void __TESTCASE_FUNC(x)()
+
+#define __BENCHMARK_FUNC(x) __benchmark_##x
+#define __BENCHMARK_TYPE(x) __BenchmarkCase_##x
+
+#define BENCHMARK_CASE(x)                                                                               \
+    static void __BENCHMARK_FUNC(x)();                                                                  \
+    struct __BENCHMARK_TYPE(x) {                                                                        \
+        __BENCHMARK_TYPE(x)                                                                             \
+        () { add_test_case_to_suite(adopt_ref(*new ::Test::TestCase(#x, __BENCHMARK_FUNC(x), true))); } \
+    };                                                                                                  \
+    static struct __BENCHMARK_TYPE(x) __BENCHMARK_TYPE(x);                                              \
+    static void __BENCHMARK_FUNC(x)()
diff --git a/Userland/Libraries/LibTest/TestMain.cpp b/Userland/Libraries/LibTest/TestMain.cpp
new file mode 100644
index 0000000000..6a232e1618
--- /dev/null
+++ b/Userland/Libraries/LibTest/TestMain.cpp
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2021, Andrew Kaster <akaster@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibTest/TestCase.h>
+
+#include <AK/Format.h>
+#include <LibTest/TestSuite.h>
+
+#ifdef KERNEL
+#    define TEST_MAIN test_main
+#else
+#    define TEST_MAIN main
+#endif
+
+int TEST_MAIN(int argc, char** argv)
+{
+    if (argc < 1 || !argv[0] || '\0' == *argv[0]) {
+        warnln("Test main does not have a valid test name!");
+        return 1;
+    }
+    int ret = ::Test::TestSuite::the().main(argv[0], argc, argv);
+    ::Test::TestSuite::release();
+    return ret;
+}
diff --git a/Userland/Libraries/LibTest/TestSuite.cpp b/Userland/Libraries/LibTest/TestSuite.cpp
new file mode 100644
index 0000000000..ac2b93b7de
--- /dev/null
+++ b/Userland/Libraries/LibTest/TestSuite.cpp
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2021, Andrew Kaster <akaster@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibTest/Macros.h> // intentionally first -- we redefine VERIFY and friends in here
+
+#include <LibCore/ArgsParser.h>
+#include <LibTest/TestSuite.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+namespace Test {
+
+TestSuite* TestSuite::s_global = nullptr;
+
+class TestElapsedTimer {
+public:
+    TestElapsedTimer() { restart(); }
+
+    void restart() { gettimeofday(&m_started, nullptr); }
+
+    u64 elapsed_milliseconds()
+    {
+        struct timeval now = {};
+        gettimeofday(&now, nullptr);
+
+        struct timeval delta = {};
+        timersub(&now, &m_started, &delta);
+
+        return delta.tv_sec * 1000 + delta.tv_usec / 1000;
+    }
+
+private:
+    struct timeval m_started = {};
+};
+
+// Declared in Macros.h
+void current_test_case_did_fail()
+{
+    TestSuite::the().current_test_case_did_fail();
+}
+
+// Declared in TestCase.h
+void add_test_case_to_suite(const NonnullRefPtr<TestCase>& test_case)
+{
+    TestSuite::the().add_case(test_case);
+}
+
+int TestSuite::main(const String& suite_name, int argc, char** argv)
+{
+    m_suite_name = suite_name;
+
+    Core::ArgsParser args_parser;
+
+    bool do_tests_only = getenv("TESTS_ONLY") != nullptr;
+    bool do_benchmarks_only = false;
+    bool do_list_cases = false;
+    const char* search_string = "*";
+
+    args_parser.add_option(do_tests_only, "Only run tests.", "tests", 0);
+    args_parser.add_option(do_benchmarks_only, "Only run benchmarks.", "bench", 0);
+    args_parser.add_option(do_list_cases, "List available test cases.", "list", 0);
+    args_parser.add_positional_argument(search_string, "Only run matching cases.", "pattern", Core::ArgsParser::Required::No);
+    args_parser.parse(argc, argv);
+
+    const auto& matching_tests = find_cases(search_string, !do_benchmarks_only, !do_tests_only);
+
+    if (do_list_cases) {
+        outln("Available cases for {}:", suite_name);
+        for (const auto& test : matching_tests) {
+            outln("    {}", test.name());
+        }
+        return 0;
+    }
+
+    outln("Running {} cases out of {}.", matching_tests.size(), m_cases.size());
+
+    return run(matching_tests);
+}
+
+NonnullRefPtrVector<TestCase> TestSuite::find_cases(const String& search, bool find_tests, bool find_benchmarks)
+{
+    NonnullRefPtrVector<TestCase> matches;
+    for (const auto& t : m_cases) {
+        if (!search.is_empty() && !t.name().matches(search, CaseSensitivity::CaseInsensitive)) {
+            continue;
+        }
+
+        if (!find_tests && !t.is_benchmark()) {
+            continue;
+        }
+        if (!find_benchmarks && t.is_benchmark()) {
+            continue;
+        }
+
+        matches.append(t);
+    }
+    return matches;
+}
+
+int TestSuite::run(const NonnullRefPtrVector<TestCase>& tests)
+{
+    size_t test_count = 0;
+    size_t test_failed_count = 0;
+    size_t benchmark_count = 0;
+    TestElapsedTimer global_timer;
+
+    for (const auto& t : tests) {
+        const auto test_type = t.is_benchmark() ? "benchmark" : "test";
+
+        warnln("Running {} '{}'.", test_type, t.name());
+        m_current_test_case_passed = true;
+
+        TestElapsedTimer timer;
+        t.func()();
+        const auto time = timer.elapsed_milliseconds();
+
+        dbgln("{} {} '{}' in {}ms", m_current_test_case_passed ? "Completed" : "Failed", test_type, t.name(), time);
+
+        if (t.is_benchmark()) {
+            m_benchtime += time;
+            benchmark_count++;
+        } else {
+            m_testtime += time;
+            test_count++;
+        }
+
+        if (!m_current_test_case_passed) {
+            test_failed_count++;
+        }
+    }
+
+    dbgln("Finished {} tests and {} benchmarks in {}ms ({}ms tests, {}ms benchmarks, {}ms other).",
+        test_count,
+        benchmark_count,
+        global_timer.elapsed_milliseconds(),
+        m_testtime,
+        m_benchtime,
+        global_timer.elapsed_milliseconds() - (m_testtime + m_benchtime));
+    dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_count - test_failed_count, test_failed_count);
+
+    return (int)test_failed_count;
+}
+
+}
diff --git a/Userland/Libraries/LibTest/TestSuite.h b/Userland/Libraries/LibTest/TestSuite.h
new file mode 100644
index 0000000000..694b6d1a69
--- /dev/null
+++ b/Userland/Libraries/LibTest/TestSuite.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2021, Andrew Kaster <akaster@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibTest/Macros.h> // intentionally first -- we redefine VERIFY and friends in here
+
+#include <AK/Format.h>
+#include <AK/Function.h>
+#include <AK/NonnullRefPtrVector.h>
+#include <AK/String.h>
+#include <LibTest/TestCase.h>
+
+namespace Test {
+
+class TestSuite {
+public:
+    static TestSuite& the()
+    {
+        if (s_global == nullptr)
+            s_global = new TestSuite();
+        return *s_global;
+    }
+
+    static void release()
+    {
+        if (s_global)
+            delete s_global;
+        s_global = nullptr;
+    }
+
+    int run(const NonnullRefPtrVector<TestCase>&);
+    int main(const String& suite_name, int argc, char** argv);
+    NonnullRefPtrVector<TestCase> find_cases(const String& search, bool find_tests, bool find_benchmarks);
+    void add_case(const NonnullRefPtr<TestCase>& test_case)
+    {
+        m_cases.append(test_case);
+    }
+
+    void current_test_case_did_fail() { m_current_test_case_passed = false; }
+
+private:
+    static TestSuite* s_global;
+    NonnullRefPtrVector<TestCase> m_cases;
+    u64 m_testtime = 0;
+    u64 m_benchtime = 0;
+    String m_suite_name;
+    bool m_current_test_case_passed = true;
+};
+
+}