diff options
author | Matthew Olsson <matthewcolsson@gmail.com> | 2020-07-03 14:36:58 -0700 |
---|---|---|
committer | Andreas Kling <kling@serenityos.org> | 2020-07-06 23:40:35 +0200 |
commit | b9cf7a833f46fb6a3add0802b6b9f65f9e70ea04 (patch) | |
tree | ee80547ab9a1907290808710bf4884e0eeda731d | |
parent | c43afe71b7dbe6ca0d76bf75631f85e463489d7c (diff) | |
download | serenity-b9cf7a833f46fb6a3add0802b6b9f65f9e70ea04.zip |
LibJS/test-js: Create test-js program, prepare for test suite refactor
This moves most of the work from run-tests.sh to test-js.cpp. This way,
we have a lot more control over how the test suite runs, as well as how
it outputs. This should result in some cool functionality!
This commit also refactors test-common.js to mimic the jest library.
This should allow tests to be much more expressive :)
-rw-r--r-- | Libraries/LibJS/Tests/test-common.js | 440 | ||||
-rw-r--r-- | Meta/Lagom/CMakeLists.txt | 6 | ||||
-rw-r--r-- | Userland/CMakeLists.txt | 1 | ||||
-rw-r--r-- | Userland/test-js.cpp | 372 |
4 files changed, 717 insertions, 102 deletions
diff --git a/Libraries/LibJS/Tests/test-common.js b/Libraries/LibJS/Tests/test-common.js index 4c7e23ac8b..d109988bca 100644 --- a/Libraries/LibJS/Tests/test-common.js +++ b/Libraries/LibJS/Tests/test-common.js @@ -1,116 +1,34 @@ -/** - * Custom error for failed assertions. - * @constructor - * @param {string} message Error message - * @returns Error - */ -function AssertionError(message) { - var instance = new Error(message); - instance.name = 'AssertionError'; - Object.setPrototypeOf(instance, Object.getPrototypeOf(this)); - return instance; -} - -/** - * Throws an `AssertionError` if `value` is not truthy. - * @param {*} value Value to be tested - */ -function assert(value) { - if (!value) - throw new AssertionError("The assertion failed!"); -} +let describe; +let test; +let expect; -/** - * Throws an `AssertionError` when called. - * @throws {AssertionError} - */ -function assertNotReached() { - throw new AssertionError("assertNotReached() was reached!"); -} +// Stores the results of each test and suite. Has a terrible +// name to avoid name collision. +let __TestResults__ = {}; -/** - * Ensures the provided functions throws a specific error. - * @param {Function} testFunction Function executing the throwing code - * @param {object} [options] - * @param {Error} [options.error] Expected error type - * @param {string} [options.name] Expected error name - * @param {string} [options.message] Expected error message - */ -function assertThrowsError(testFunction, options) { - try { - testFunction(); - assertNotReached(); - } catch (e) { - if (options.error !== undefined) - assert(e instanceof options.error); - if (options.name !== undefined) - assert(e.name === options.name); - if (options.message !== undefined) - assert(e.message === options.message); - } -} - -/** - * Ensures the provided JavaScript source code results in a SyntaxError - * @param {string} source The JavaScript source code to compile - */ -function assertIsSyntaxError(source) { - assertThrowsError(() => { - new Function(source)(); - }, { - error: SyntaxError, - }); -} - -/** - * Ensures the provided arrays contain exactly the same items. - * @param {Array} a First array - * @param {Array} b Second array - */ -function assertArrayEquals(a, b) { - if (a.length != b.length) - throw new AssertionError("Array lengths do not match"); - - for (var i = 0; i < a.length; i++) { - if (a[i] !== b[i]) - throw new AssertionError("Elements do not match"); - } -} +// This array is used to communicate with the C++ program. It treats +// each message in this array as a separate message. Has a terrible +// name to avoid name collision. +let __UserOutput__ = []; -const assertVisitsAll = (testFunction, expectedOutput) => { - const visited = []; - testFunction(value => visited.push(value)); - assert(visited.length === expectedOutput.length); - expectedOutput.forEach((value, i) => assert(visited[i] === value)); +// We also rebind console.log here to use the array above +console.log = (...args) => { + __UserOutput__.push(args.join(" ")); }; -/** - * Check whether the difference between two numbers is less than 0.000001. - * @param {Number} a First number - * @param {Number} b Second number - */ -function isClose(a, b) { - return Math.abs(a - b) < 0.000001; -} - -/** - * Quick and dirty deep equals method. - * @param {*} a First value - * @param {*} b Second value - */ -function assertDeepEquals(a, b) { - assert(deepEquals(a, b)); -} +// Use an IIFE to avoid polluting the global namespace as much as possible +(() => { -function deepEquals(a, b) { +// FIXME: This is a very naive deepEquals algorithm +const deepEquals = (a, b) => { if (Array.isArray(a)) return Array.isArray(b) && deepArrayEquals(a, b); if (typeof a === "object") return typeof b === "object" && deepObjectEquals(a, b); - return a === b; + return Object.is(a, b); } -function deepArrayEquals(a, b) { +const deepArrayEquals = (a, b) => { if (a.length !== b.length) return false; for (let i = 0; i < a.length; ++i) { @@ -120,7 +38,7 @@ function deepArrayEquals(a, b) { return true; } -function deepObjectEquals(a, b) { +const deepObjectEquals = (a, b) => { if (a === null) return b === null; for (let key of Reflect.ownKeys(a)) { @@ -129,3 +47,321 @@ function deepObjectEquals(a, b) { } return true; } + +class ExpectationError extends Error { + constructor(message, fileName, lineNumber) { + super(message, fileName, lineNumber); + this.name = "ExpectationError"; + } +} + +class Expector { + constructor(target, inverted) { + this.target = target; + this.inverted = !!inverted; + } + + get not() { + return new Expector(this.target, !this.inverted); + } + + toBe(value) { + this.__doMatcher(() => { + this.__expect(Object.is(this.target, value)); + }); + } + + toHaveLength(length) { + this.__doMatcher(() => { + this.__expect(Object.is(this.target.length, length)); + }); + } + + toHaveProperty(property, value) { + this.__doMatcher(() => { + let object = this.target; + + if (typeof property === "string" && property.includes(".")) { + let propertyArray = []; + + while (true) { + let index = property.indexOf("."); + if (index === -1) { + propertyArray.push(property); + break; + } + + propertyArray.push(property.substring(0, index)); + property = property.substring(index, property.length); + } + + property = propertyArray; + } + + if (Array.isArray(property)) { + for (let key of property) { + if (object === undefined || object === null) { + if (this.inverted) + return; + throw new ExpectationError(); + } + object = object[key]; + } + } else { + object = object[property]; + } + + this.__expect(object !== undefined); + if (value !== undefined) + this.__expect(deepEquals(object, value)); + }); + } + + toBeCloseTo(number, numDigits) { + if (numDigits === undefined) + numDigits = 2; + + this.__doMatcher(() => { + this.__expect(Math.abs(number - this.target) < (10 ** -numDigits / numDigits)); + }); + } + + toBeDefined() { + this.__doMatcher(() => { + this.__expect(this.target !== undefined); + }); + } + + toBeFalsey() { + this.__doMatcher(() => { + this.__expect(!this.target); + }); + } + + toBeGreaterThan(number) { + this.__doMatcher(() => { + this.__expect(this.target > number); + }); + } + + toBeGreaterThanOrEqual(number) { + this.__doMatcher(() => { + this.__expect(this.target >= number); + }); + } + + toBeLessThan(number) { + this.__doMatcher(() => { + this.__expect(this.target < number); + }); + } + + toBeLessThanOrEqual(number) { + this.__doMatcher(() => { + this.__expect(this.target <= number); + }); + } + + toBeInstanceOf(class_) { + this.__doMatcher(() => { + this.__expect(this.target instanceof class_); + }); + } + + toBeNull() { + this.__doMatcher(() => { + this.__expect(this.target === null); + }); + } + + toBeTruthy() { + this.__doMatcher(() => { + this.__expect(!!this.target); + }); + } + + toBeUndefined() { + this.__doMatcher(() => { + this.__expect(this.target === undefined); + }); + } + + toBeNaN() { + this.__doMatcher(() => { + this.__expect(isNaN(this.target)); + }); + } + + toContain(item) { + this.__doMatcher(() => { + // FIXME: Iterator check + for (let element of this.target) { + if (item === element) + return; + } + + throw new ExpectationError(); + }); + } + + toContainEqual(item) { + this.__doMatcher(() => { + // FIXME: Iterator check + for (let element of this.target) { + if (deepEquals(item, element)) + return; + } + + throw new ExpectationError(); + }); + } + + toEqual(value) { + this.__doMatcher(() => { + this.__expect(deepEquals(this.target, value)); + }); + } + + toThrow(value) { + this.__expect(typeof this.target === "function"); + this.__expect(typeof value === "string" || typeof value === "function" || value === undefined); + + this.__doMatcher(() => { + try { + this.target(); + this.__expect(false); + } catch (e) { + if (typeof value === "string") { + this.__expect(e.message.includes(value)); + } else if (typeof value === "function") { + this.__expect(e instanceof value); + } + } + }); + } + + pass(message) { + // FIXME: This does nothing. If we want to implement things + // like assertion count, this will have to do something + } + + // jest-extended + fail(message) { + // FIXME: message is currently ignored + this.__doMatcher(() => { + this.__expect(false); + }) + } + + // jest-extended + toThrowWithMessage(class_, message) { + this.__expect(typeof this.target === "function"); + this.__expect(class_ !== undefined); + this.__expect(message !== undefined); + + this.__doMatcher(() => { + try { + this.target(); + this.__expect(false); + } catch (e) { + this.__expect(e instanceof class_); + this.__expect(e.message.includes(message)); + } + }); + } + + // Test for syntax errors; target must be a string + toEval() { + this.__expect(typeof this.target === "string"); + + if (!this.inverted) { + try { + new Function(this.target)(); + } catch (e) { + throw new ExpectationError(); + } + } else { + try { + new Function(this.target)(); + throw new ExpectationError(); + } catch (e) { + if (e.name !== "SyntaxError") + throw new ExpectationError(); + } + } + } + + // Must compile regardless of inverted-ness + toEvalTo(value) { + this.__expect(typeof this.target === "string"); + + let result; + + try { + result = new Function(this.target)(); + } catch (e) { + throw new ExpectationError(); + } + + this.__doMatcher(() => { + this.__expect(deepEquals(value, result)); + }); + } + + __doMatcher(matcher) { + if (!this.inverted) { + matcher(); + } else { + let threw = false; + try { + matcher(); + } catch (e) { + if (e.name === "ExpectationError") + threw = true; + } + if (!threw) + throw new ExpectationError(); + } + } + + __expect(value) { + if (value !== true) + throw new ExpectationError(); + } +} + +expect = value => new Expector(value); + +// describe is able to lump test results inside of it by using this context +// variable. Top level tests are assumed to be in the default context +const defaultSuiteMessage = "__$$TOP_LEVEL$$__"; +let suiteMessage = defaultSuiteMessage; + +describe = (message, callback) => { + suiteMessage = message; + callback(); + suiteMessage = defaultSuiteMessage; +} + +test = (message, callback) => { + if (!__TestResults__[suiteMessage]) + __TestResults__[suiteMessage] = {}; + + const suite = __TestResults__[suiteMessage]; + + if (!suite[message]) + suite[message] = {}; + + try { + callback(); + suite[message] = { + passed: true, + }; + } catch (e) { + suite[message] = { + passed: false, + }; + } +} + +})(); diff --git a/Meta/Lagom/CMakeLists.txt b/Meta/Lagom/CMakeLists.txt index e54e10482a..544ba7611e 100644 --- a/Meta/Lagom/CMakeLists.txt +++ b/Meta/Lagom/CMakeLists.txt @@ -75,6 +75,12 @@ if (BUILD_LAGOM) target_link_libraries(js_lagom stdc++) target_link_libraries(js_lagom pthread) + add_executable(test-js_lagom ../../Userland/test-js.cpp) + set_target_properties(test-js_lagom PROPERTIES OUTPUT_NAME test-js) + target_link_libraries(test-js_lagom Lagom) + target_link_libraries(test-js_lagom stdc++) + target_link_libraries(test-js_lagom pthread) + add_executable(test-crypto_lagom ../../Userland/test-crypto.cpp) set_target_properties(test-crypto_lagom PROPERTIES OUTPUT_NAME test-crypto) target_link_libraries(test-crypto_lagom Lagom) diff --git a/Userland/CMakeLists.txt b/Userland/CMakeLists.txt index ecfcff4943..ee3ad79be7 100644 --- a/Userland/CMakeLists.txt +++ b/Userland/CMakeLists.txt @@ -32,4 +32,5 @@ target_link_libraries(pape LibGUI) target_link_libraries(paste LibGUI) target_link_libraries(pro LibProtocol) target_link_libraries(test-crypto LibCrypto LibTLS LibLine) +target_link_libraries(test-js LibJS LibLine LibCore) target_link_libraries(tt LibPthread) diff --git a/Userland/test-js.cpp b/Userland/test-js.cpp new file mode 100644 index 0000000000..6eeb615a8e --- /dev/null +++ b/Userland/test-js.cpp @@ -0,0 +1,372 @@ +/* + * Copyright (c) 2020, Matthew Olsson <matthewcolsson@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <AK/JsonValue.h> +#include <AK/JsonObject.h> +#include <AK/LogStream.h> +#include <LibCore/File.h> +#include <LibJS/Interpreter.h> +#include <LibJS/Lexer.h> +#include <LibJS/Parser.h> +#include <LibJS/Runtime/Array.h> +#include <LibJS/Runtime/GlobalObject.h> +#include <LibJS/Runtime/MarkedValueList.h> +#include <sys/time.h> +#include <stdlib.h> +#include <string.h> + +#define TOP_LEVEL_TEST_NAME "__$$TOP_LEVEL$$__" + +// FIXME: Will eventually not be necessary when all tests are converted +Vector<String> tests_to_run = { + "add-values-to-primitive.js", + "automatic-semicolon-insertion.js", + "comments-basic.js", + "debugger-statement.js", + "empty-statements.js", + "exception-ReferenceError.js", + "exponentiation-basic.js", + "indexed-access-string-object.js", + "invalid-lhs-in-assignment.js", + "tagged-template-literals.js", + "switch-basic.js", + "update-expression-on-member-expression.js", +}; + +struct FileTest { + String name; + bool passed; +}; + +struct FileSuite { + String name; + int passed { 0 }; + int failed { 0 }; + Vector<FileTest> tests {}; +}; + +struct TestError { + JS::Parser::Error error; + String hint; +}; + +struct FileResults { + String file; + Optional<TestError> error {}; + int passed { 0 }; + int failed { 0 }; + Vector<FileSuite> suites {}; +}; + +struct Results { + Vector<FileResults> file_results {}; +}; + +Optional<TestError> parse_and_run_file(JS::Interpreter& interpreter, const String& path) +{ + auto file = Core::File::construct(path); + auto result = file->open(Core::IODevice::ReadOnly); + ASSERT(result); + + auto contents = file->read_all(); + String test_file_string(reinterpret_cast<const char*>(contents.data()), contents.size()); + file->close(); + + auto parser = JS::Parser(JS::Lexer(test_file_string)); + auto program = parser.parse_program(); + + if (parser.has_errors()) { + auto error = parser.errors()[0]; + return TestError { error, error.source_location_hint(test_file_string) }; + } else { + interpreter.run(interpreter.global_object(), *program); + } + + return {}; +} + +FileResults run_test(const String& path, const String& test_root) +{ + auto interpreter = JS::Interpreter::create<JS::GlobalObject>(); + + if (parse_and_run_file(*interpreter, String::format("%s/test-common.js", test_root.characters())).has_value()) { + dbg() << "test-common.js failed to parse"; + exit(1); + } + + auto source_file_result = parse_and_run_file(*interpreter, String::format("%s/%s", test_root.characters(), path.characters())); + if (source_file_result.has_value()) + return { path, source_file_result }; + + // Print any output + // FIXME: Should be printed to stdout in a nice format + auto& arr = interpreter->get_variable("__UserOutput__", interpreter->global_object()).as_array(); + for (auto& entry : arr.indexed_properties()) { + dbg() << "OUTPUT: " << entry.value_and_attributes(&interpreter->global_object()).value.to_string(*interpreter); + } + + // FIXME: This is _so_ scuffed + auto result = interpreter->get_variable("__TestResults__", interpreter->global_object()); + auto json_object = interpreter->get_variable("JSON", interpreter->global_object()); + auto stringify = json_object.as_object().get("stringify"); + JS::MarkedValueList arguments(interpreter->heap()); + arguments.append(result); + auto json_string = interpreter->call(stringify.as_function(), interpreter->this_value(interpreter->global_object()), move(arguments)).to_string(*interpreter); + + auto json_result = JsonValue::from_string(json_string); + + if (!json_result.has_value()) { + dbg() << "BAD JSON:"; + dbg() << json_string; + return {}; + } + + auto json = json_result.value(); + + FileResults results { path }; + + json.as_object().for_each_member([&](const String& property, const JsonValue& value) { + FileSuite suite { property }; + + value.as_object().for_each_member([&](const String& property1, const JsonValue& value1) { + FileTest test { property1, false }; + + if (value1.is_object()) { + auto obj = value1.as_object(); + if (obj.has("passed")) { + auto passed = obj.get("passed"); + test.passed = passed.is_bool() && passed.as_bool(); + } + } + + if (test.passed) { + suite.passed++; + } else { + suite.failed++; + } + + suite.tests.append(test); + }); + + if (suite.failed) { + results.failed++; + } else { + results.passed++; + } + + results.suites.append(suite); + }); + + return results; +} + +bool skip_test(char* test_name) +{ + return !strcmp(test_name, "test-common.js") || !strcmp(test_name, "run_tests.sh"); +} + +enum Modifier { + BG_RED, + BG_GREEN, + FG_RED, + FG_GREEN, + FG_GRAY, + FG_BLACK, + FG_BOLD, + CLEAR, +}; + +void print_modifiers(Vector<Modifier> modifiers) +{ + for (auto& modifier : modifiers) { + auto code = [&]() -> String { + switch (modifier) { + case BG_RED: + return "\033[48;2;255;0;102m"; + case BG_GREEN: + return "\033[48;2;102;255;0m"; + case FG_RED: + return "\033[38;2;255;0;102m"; + case FG_GREEN: + return "\033[38;2;102;255;0m"; + case FG_GRAY: + return "\033[38;2;135;139;148m"; + case FG_BLACK: + return "\033[30m"; + case FG_BOLD: + return "\033[1m"; + case CLEAR: + return "\033[0m"; + } + ASSERT_NOT_REACHED(); + }; + printf("%s", code().characters()); + } +} + +void print_file_results(const FileResults& results) +{ + if (results.failed || results.error.has_value()) { + print_modifiers({ BG_RED, FG_BLACK, FG_BOLD }); + printf(" FAIL "); + print_modifiers({ CLEAR }); + } else { + print_modifiers({ BG_GREEN, FG_BLACK, FG_BOLD }); + printf(" PASS "); + print_modifiers({ CLEAR }); + } + + printf(" %s\n", results.file.characters()); + + if (results.error.has_value()) { + auto test_error = results.error.value(); + + print_modifiers({ FG_RED }); + printf(" ❌ The file failed to parse\n\n"); + print_modifiers({ FG_GRAY }); + for (auto& message : test_error.hint.split('\n', true)) { + printf(" %s\n", message.characters()); + } + print_modifiers({ FG_RED }); + printf(" %s\n\n", test_error.error.to_string().characters()); + + return; + } + + if (results.failed) { + for (auto& suite : results.suites) { + if (!suite.failed) + continue; + + bool top_level = suite.name == TOP_LEVEL_TEST_NAME; + + if (!top_level) { + print_modifiers({ FG_GRAY, FG_BOLD }); + printf(" ❌ Suite: "); + print_modifiers({ CLEAR, FG_RED }); + printf("%s\n", suite.name.characters()); + print_modifiers({ CLEAR }); + } + + for (auto& test : suite.tests) { + if (test.passed) + continue; + + if (!top_level) { + print_modifiers({ FG_GRAY, FG_BOLD }); + printf(" Test: "); + print_modifiers({ CLEAR, FG_RED }); + printf("%s\n", test.name.characters()); + print_modifiers({ CLEAR }); + } else { + print_modifiers({ FG_GRAY, FG_BOLD }); + printf(" ❌ Test: "); + print_modifiers({ CLEAR, FG_RED }); + printf("%s\n", test.name.characters()); + print_modifiers({ CLEAR }); + } + } + } + } +} + +void print_results(const Results& results, double time_elapsed) +{ + for (auto& result : results.file_results) + print_file_results(result); + + int suites_passed = 0; + int suites_failed = 0; + int tests_passed = 0; + int tests_failed = 0; + + for (auto& file_result : results.file_results) { + for (auto& suite : file_result.suites) { + tests_passed += suite.passed; + tests_failed += suite.failed; + + if (suite.failed) { + suites_failed++; + } else { + suites_passed++; + } + } + } + + + printf("\nTest Suites: "); + if (suites_failed) { + print_modifiers({ FG_RED }); + printf("%d failed, ", suites_failed); + print_modifiers({ CLEAR }); + } + if (suites_passed) { + print_modifiers({ FG_GREEN }); + printf("%d passed, ", suites_passed); + print_modifiers({ CLEAR }); + } + printf("%d total\n", suites_failed + suites_passed); + + printf("Tests: "); + if (tests_failed) { + print_modifiers({ FG_RED }); + printf("%d failed, ", tests_failed); + print_modifiers({ CLEAR }); + } + if (tests_passed) { + print_modifiers({ FG_GREEN }); + printf("%d passed, ", tests_passed); + print_modifiers({ CLEAR }); + } + printf("%d total\n", tests_failed + tests_passed); + + printf("Time: %-.3fs\n\n", time_elapsed); +} + +double get_time() +{ + struct timeval tv1; + struct timezone tz1; + auto return_code = gettimeofday(&tv1, &tz1); + ASSERT(return_code >= 0); + return static_cast<double>(tv1.tv_sec) + static_cast<double>(tv1.tv_usec) / 1'000'000; +} + +int main(int, char** argv) +{ + String test_root = argv[1]; + Results results; + + double start_time = get_time(); + + for (auto& test : tests_to_run) + results.file_results.append(run_test(test, test_root)); + + print_results(results, get_time() - start_time); + + return 0; +} + |