Base/CI: Run tests from /usr/Tests in GitHub Actions

Uncomment the tests that were disabled due to frequent freezes when
running without KVM. This also adds a new github actions group for
every single test, which makes it easier to browse test boundaries
during test runs.

Move catting the serial output log back to its own step, so that it
has higher visibility. The previous solution was also shown to not
actually cat the log in the case of a failed boot and timeout :^(.

2 files changed, 39 insertions, 20 deletions

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index fcd97984e1..259a973611 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -141,24 +141,20 @@ jobs:
       if: ${{ matrix.debug-macros == 'NORMAL_DEBUG'}}
       working-directory: ${{ github.workspace }}/Build
       env:
+        SERENITY_QEMU_CPU: "max,vmx=off"
         SERENITY_KERNEL_CMDLINE: "boot_mode=self-test"
         SERENITY_RUN: "ci"
       run: |
         echo "::group::ninja run # Qemu output"
-        run_worked=y
-        ninja run || run_worked=n
+        ninja run
         echo "::endgroup::"
-        echo "::group::cat debug.log # Serenity output"
-        if [ "y" = "${run_worked}" ] ; then
-          cat debug.log
-        else
-          echo "skipped (qemu had non-zero exit-code)"
-        fi
-        echo "::endgroup::"
-        [ "y" = ${run_worked} ]
-      timeout-minutes: 10
-      # FIXME: When stable, remove continue on error. (See issue #5541)
-      continue-on-error: true
+      timeout-minutes: 20
+
+    - name: Print Target Logs
+      # Extremely useful if Serenity hangs trying to run one of the tests
+      if: ${{ !cancelled() && matrix.debug-macros == 'NORMAL_DEBUG'}}
+      working-directory: ${{ github.workspace }}/Build
+      run: '[ ! -e debug.log ] || cat debug.log'
 
   build_and_test_lagom:
     runs-on: ${{ matrix.os }}
diff --git a/Base/home/anon/tests/run-tests-and-shutdown.sh b/Base/home/anon/tests/run-tests-and-shutdown.sh
index 82fd6d214e..0abd0a86f6 100755
--- a/Base/home/anon/tests/run-tests-and-shutdown.sh
+++ b/Base/home/anon/tests/run-tests-and-shutdown.sh
@@ -6,26 +6,49 @@ echo "==== Running Tests on SerenityOS ===="
 run(index) {
     shift
     test_cmd=($*)
-    echo "Running test $index out of $count_of_all_tests -- $test_cmd"
+    echo "==== Running test $index out of $count_of_all_tests -- $test_cmd ===="
+    echo "::group::$test_cmd"
     if $test_cmd {
-        echo "::debug file=$test_cmd:: $test_cmd passed!"
+        echo "::endgroup::"
+        echo "==== $test_cmd passed! ===="
     } else {
+        echo "::endgroup::"
+        failed_tests=${concat_lists $failed_tests ($test_cmd)}
+        echo "==== Added $test_cmd to list of failed tests. Failed tests so far: $failed_tests ===="
         echo "::error file=$test_cmd:: $test_cmd returned non-zero exit code, check logs!"
     }
 }
 
-# TODO: test-web requires the window server
-system_tests=((test-js --show-progress=false) test-pthread /usr/Tests/LibM/test-math (test-crypto -t bigint))
-# FIXME: Running too much at once is likely to run into #5541. Remove commented out find below when stable
-all_tests=${concat_lists $system_tests} #$(find /usr/Tests -type f | grep -v Kernel | grep -v .inc | shuf))
+# Files in /usr/Tests/* that we don't want to execute match these patterns:
+#     Kernel/Legacy: Kernel tests that are either old patched exploits, or hang the system/shell
+#     .inc: Shell test helper file that's not a test
+#     UserEmulator: Tests designed to run inside the Userspace Emulator
+#     stack-smash: Intentionally crashes by smashing the stack
+#     TestJSON: AK/TestJSON makes assumptions about $PWD to load its input files
+#     .frm: Test inputs that are not tests
+#     test-web: Requires the window server in order to work
+#     test-js: We start this one manually with the show progress flag set to false
+exclude_patterns='Kernel/Legacy|.inc|UserEmulator|stack-smash|TestJSON|.frm|test-web|test-js'
+
+system_tests=((test-js --show-progress=false) (test-crypto -c -t test))
+all_tests=${concat_lists $system_tests $(find /usr/Tests -type f | grep -E -v $exclude_patterns | shuf) }
 count_of_all_tests=${length $all_tests}
+failed_tests=()
 
 for index i cmd in $all_tests {
     run $(expr $i + 1) $cmd
 }
 
-echo "==== Done running tests ===="
+fail_count=${length $failed_tests}
+
+echo "==== Out of $count_of_all_tests tests, $(expr $count_of_all_tests - $fail_count) passed and $fail_count failed. ===="
+
+if test $fail_count -gt 0 {
+    echo "==== Failing tests: $failed_tests ===="
+}
 
 if test $DO_SHUTDOWN_AFTER_TESTS {
     shutdown -n
 }
+
+exit $fail_count