diff options
Diffstat (limited to 'tests/sytest/are-we-synapse-yet.py')
-rwxr-xr-x | tests/sytest/are-we-synapse-yet.py | 38 |
1 files changed, 22 insertions, 16 deletions
diff --git a/tests/sytest/are-we-synapse-yet.py b/tests/sytest/are-we-synapse-yet.py index 0b334ba..3d21fa4 100755 --- a/tests/sytest/are-we-synapse-yet.py +++ b/tests/sytest/are-we-synapse-yet.py @@ -11,7 +11,7 @@ import sys # The main complexity is grouping tests sensibly into features like 'Registration' # and 'Federation'. Then it just checks the ones which are passing and calculates # percentages for each group. Produces results like: -# +# # Client-Server APIs: 29% (196/666 tests) # ------------------- # Registration : 62% (20/32 tests) @@ -28,11 +28,13 @@ import sys # ✓ POST /register can create a user # ✓ POST /register downcases capitals in usernames # ... -# +# # You can also tack `-v` on to see exactly which tests each category falls under. test_mappings = { "nsp": "Non-Spec API", + "unk": "Unknown API (no group specified)", + "app": "Application Services API", "f": "Federation", # flag to mark test involves federation "federation_apis": { @@ -50,6 +52,7 @@ test_mappings = { "fpb": "Public Room API", "fdk": "Device Key APIs", "fed": "Federation API", + "fsd": "Send-to-Device APIs", }, "client_apis": { @@ -61,6 +64,8 @@ test_mappings = { "pro": "Profile", "dev": "Devices", "dvk": "Device Keys", + "dkb": "Device Key Backup", + "xsk": "Cross-signing Keys", "pre": "Presence", "crm": "Create Room", "syn": "Sync API", @@ -98,7 +103,7 @@ test_mappings = { "adm": "Server Admin API", "ign": "Ignore Users", "udr": "User Directory APIs", - "app": "Application Services API", + "jso": "Enforced canonical JSON", }, } @@ -156,20 +161,22 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): total_tests = 0 for gid, tests in gid_to_tests.items(): group_total = len(tests) + if group_total == 0: + continue group_passing = 0 test_names_and_marks = [] for name, passing in tests.items(): if passing: group_passing += 1 test_names_and_marks.append(f"{'✓' if passing else '×'} {name}") - + total_tests += group_total total_passing += group_passing pct = "{0:.0f}%".format(group_passing/group_total * 100) line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total) subsections.append(line) subsection_test_names[line] = test_names_and_marks - + pct = "{0:.0f}%".format(total_passing/total_tests * 100) print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) print("-" * (len(header_name)+1)) @@ -186,7 +193,6 @@ def main(results_tap_path, verbose): test_name_to_group_id = {} fed_tests = set() client_tests = set() - groupless_tests = set() with open("./are-we-synapse-yet.list", "r") as f: for line in f.readlines(): test_name = " ".join(line.split(" ")[1:]).strip() @@ -212,8 +218,12 @@ def main(results_tap_path, verbose): # test_name: OK # } }, + "appservice": { + "app": {}, + }, "nonspec": { - "nsp": {} + "nsp": {}, + "unk": {} }, } with open(results_tap_path, "r") as f: @@ -224,10 +234,11 @@ def main(results_tap_path, verbose): name = test_result["name"] group_id = test_name_to_group_id.get(name) if not group_id: - groupless_tests.add(name) - # raise Exception("The test '%s' doesn't have a group" % (name,)) + summary["nonspec"]["unk"][name] = test_result["ok"] if group_id == "nsp": summary["nonspec"]["nsp"][name] = test_result["ok"] + elif group_id == "app": + summary["appservice"]["app"][name] = test_result["ok"] elif group_id in test_mappings["federation_apis"]: group = summary["federation"].get(group_id, {}) group[name] = test_result["ok"] @@ -243,12 +254,7 @@ def main(results_tap_path, verbose): print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose) print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose) print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose) - if verbose: - print("The following tests don't have a group:") - for name in groupless_tests: - print(" %s" % (name,)) - else: - print("%d tests don't have a group" % len(groupless_tests)) + print_stats("Application Services APIs", summary["appservice"], test_mappings, verbose) @@ -257,4 +263,4 @@ if __name__ == '__main__': parser.add_argument("tap_file", help="path to results.tap") parser.add_argument("-v", action="store_true", help="show individual test names in output") args = parser.parse_args() - main(args.tap_file, args.v) + main(args.tap_file, args.v)
\ No newline at end of file |