Meta: Generate failing tests for unsupported assertions in wasm tests

Let's not ignore a test just because our test generator doesn't
understand it, assume the worst and generate a failing test case
instead.

1 files changed, 29 insertions, 2 deletions

diff --git a/Meta/generate-libwasm-spec-test.py b/Meta/generate-libwasm-spec-test.py
index dc968bf323..8e0b63ea62 100644
--- a/Meta/generate-libwasm-spec-test.py
+++ b/Meta/generate-libwasm-spec-test.py
@@ -98,7 +98,19 @@ def generate(ast):
                     "result": parse_typed_value(entry[2]) if len(entry) == 3 else None
                 })
             else:
-                print("Ignoring unknown assertion argument", entry[1][0], file=stderr)
+                if not len(tests):
+                    tests.append({
+                        "module": "",
+                        "tests": []
+                    })
+                tests[-1]["tests"].append({
+                    "kind": "testgen_fail",
+                    "function": {
+                        "name": "<unknown>",
+                        "args": []
+                    },
+                    "reason": f"Unknown assertion {entry[0][0][len('assert_'):]}"
+                })
         elif len(entry) >= 2 and entry[0][0] == 'invoke':
             # toplevel invoke :shrug:
             tests[-1]["tests"].append({
@@ -110,7 +122,19 @@ def generate(ast):
                 "result": parse_typed_value(entry[2]) if len(entry) == 3 else None
             })
         else:
-            print("Ignoring unknown entry", entry, file=stderr)
+            if not len(tests):
+                tests.append({
+                    "module": "",
+                    "tests": []
+                })
+            tests[-1]["tests"].append({
+                "kind": "testgen_fail",
+                "function": {
+                    "name": "<unknown>",
+                    "args": []
+                },
+                "reason": f"Unknown command {entry[0][0]}"
+            })
     return tests
 
 
@@ -178,6 +202,9 @@ def genresult(ident, entry):
     if entry['kind'] == 'ignore':
         return f'module.invoke({ident}, {", ".join(genarg(x) for x in entry["function"]["args"])});\n    '
 
+    if entry['kind'] == 'testgen_fail':
+        return f'throw Exception("Test Generator Failure: " + {json.dumps(entry["reason"])});\n    '
+
     return f'throw Exception("(Test Generator) Unknown test kind {entry["kind"]}");\n    '