From 2f5fe79988b1f6296f2bb70102bf8f77b4f9fec7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 8 May 2026 10:01:07 -0700 Subject: [PATCH 1/5] Testing: Add verbose logging feature, and stop excessive spec logging --- check.py | 6 +++--- scripts/test/shared.py | 16 ++++++++++++---- scripts/test/support.py | 4 +++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/check.py b/check.py index a670bb96001..3e503925348 100755 --- a/check.py +++ b/check.py @@ -191,7 +191,7 @@ def run_opt_test(wast, stdout=None): def check_expected(actual, expected, stdout=None): if expected and os.path.exists(expected): expected = open(expected).read() - print(' (using expected output)', file=stdout) + shared.verbose_log(' (using expected output)', file=stdout) actual = actual.strip() expected = expected.strip() if actual != expected: @@ -228,7 +228,7 @@ def run_one_spec_test(wast: Path, stdout=None): actual = run_spec_test(str(wast), stdout=stdout) except Exception as e: if ('wasm-validator error' in str(e) or 'error: ' in str(e)) and '.fail.' in test_name: - print('<< test failed as expected >>', file=stdout) + shared.verbose_log('<< test failed as expected >>', file=stdout) return # don't try all the binary format stuff TODO else: shared.fail_with_error(str(e)) @@ -248,7 +248,7 @@ def run_one_spec_test(wast: Path, stdout=None): if not module: # Skip any initial assertions that don't have a module continue - print(f' testing split module {i}', file=stdout) + shared.verbose_log(f' testing split module {i}', file=stdout) split_name = base_name + f'_split{i}.wast' support.write_wast(split_name, module) run_opt_test(split_name, stdout=stdout) # also that our optimizer doesn't break on it diff --git a/scripts/test/shared.py b/scripts/test/shared.py index 4a41a046bde..85ea5d7bda8 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -97,6 +97,9 @@ def parse_args(args): action='store_false', default=True, help='Disables the automatic selection of important initial contents ' 'in fuzzer.') + parser.add_argument( + '--verbose', action='store_true', default=False, + help='Enables verbose logging.') return parser.parse_args(args) @@ -114,6 +117,11 @@ def warn(text): print('warning:', text, file=sys.stderr) +def verbose_log(*args, **kwargs): + if options.verbose: + print(*args, **kwargs) + + # setup # Locate Binaryen build artifacts directory (bin/ by default) @@ -468,16 +476,16 @@ def binary_format_check(wast, verify_final_result=True, base_name=None, stdout=N as_file = f"{base_name}-a.wasm" if base_name is not None else "a.wasm" disassembled_file = f"{base_name}-ab.wast" if base_name is not None else "ab.wast" - print(' (binary format check)', file=stdout) + verbose_log(' (binary format check)', file=stdout) cmd = WASM_AS + [wast, '-o', as_file, '-all', '-g'] - print(' ', ' '.join(cmd), file=stdout) + verbose_log(' ', ' '.join(cmd), file=stdout) if os.path.exists(as_file): os.unlink(as_file) subprocess.check_call(cmd, stdout=subprocess.PIPE) assert os.path.exists(as_file) cmd = WASM_DIS + [as_file, '-o', disassembled_file, '-all'] - print(' ', ' '.join(cmd), file=stdout) + verbose_log(' ', ' '.join(cmd), file=stdout) if os.path.exists(disassembled_file): os.unlink(disassembled_file) subprocess.check_call(cmd, stdout=subprocess.PIPE) @@ -485,7 +493,7 @@ def binary_format_check(wast, verify_final_result=True, base_name=None, stdout=N # make sure it is a valid wast cmd = WASM_OPT + [disassembled_file, '-all', '-q'] - print(' ', ' '.join(cmd), file=stdout) + verbose_log(' ', ' '.join(cmd), file=stdout) subprocess.check_call(cmd, stdout=subprocess.PIPE) if verify_final_result: diff --git a/scripts/test/support.py b/scripts/test/support.py index c79bf26ae6c..0cfcfaf7c53 100644 --- a/scripts/test/support.py +++ b/scripts/test/support.py @@ -16,6 +16,8 @@ import re import subprocess +from . import shared + QUOTED = re.compile(r'\(module\s*(\$\S*)?\s+(quote|binary)') MODULE_DEFINITION_OR_INSTANCE = re.compile(r'(?m)\(module\s+(instance|definition)') @@ -147,7 +149,7 @@ def run_command(cmd, expected_status=0, stdout=None, stderr=None, assert stderr == subprocess.PIPE or stderr is None, \ "Can't redirect stderr if using expected_err" stderr = subprocess.PIPE - print('executing: ', ' '.join(cmd), file=stdout) + shared.verbose_log('executing: ', ' '.join(cmd), file=stdout) out, err, code = _subprocess_run(cmd, stdout=subprocess.PIPE, stderr=stderr, encoding='UTF-8') From d114c3b6e0d35954b33aee6bf1f2f592bb08a713 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 8 May 2026 10:02:36 -0700 Subject: [PATCH 2/5] fix --- scripts/test/shared.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test/shared.py b/scripts/test/shared.py index e066a92deec..d8211d8143a 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -120,6 +120,7 @@ def warn(text): def print_heading(msg): print(f'[ {msg} ]') + def verbose_log(*args, **kwargs): if options.verbose: print(*args, **kwargs) From 134b87eb2eddeae929ed72089893c7a0771f4aaa Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 8 May 2026 11:56:42 -0700 Subject: [PATCH 3/5] feedback --- check.py | 6 +++--- scripts/test/shared.py | 8 ++++---- scripts/test/support.py | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/check.py b/check.py index d1e1f49103a..5c6b9ef4c90 100755 --- a/check.py +++ b/check.py @@ -192,7 +192,7 @@ def run_opt_test(wast, stdout=None): def check_expected(actual, expected, stdout=None): if expected and os.path.exists(expected): expected = open(expected).read() - shared.verbose_log(' (using expected output)', file=stdout) + shared.verbose_log(' (using expected output)') actual = actual.strip() expected = expected.strip() if actual != expected: @@ -229,7 +229,7 @@ def run_one_spec_test(wast: Path, stdout=None): actual = run_spec_test(str(wast), stdout=stdout) except Exception as e: if ('wasm-validator error' in str(e) or 'error: ' in str(e)) and '.fail.' in test_name: - shared.verbose_log('<< test failed as expected >>', file=stdout) + shared.verbose_log('<< test failed as expected >>') return # don't try all the binary format stuff TODO else: shared.fail_with_error(str(e)) @@ -249,7 +249,7 @@ def run_one_spec_test(wast: Path, stdout=None): if not module: # Skip any initial assertions that don't have a module continue - shared.verbose_log(f' testing split module {i}', file=stdout) + shared.verbose_log(f' testing split module {i}') split_name = base_name + f'_split{i}.wast' support.write_wast(split_name, module) run_opt_test(split_name, stdout=stdout) # also that our optimizer doesn't break on it diff --git a/scripts/test/shared.py b/scripts/test/shared.py index d8211d8143a..492b90b4454 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -480,16 +480,16 @@ def binary_format_check(wast, verify_final_result=True, base_name=None, stdout=N as_file = f"{base_name}-a.wasm" if base_name is not None else "a.wasm" disassembled_file = f"{base_name}-ab.wast" if base_name is not None else "ab.wast" - verbose_log(' (binary format check)', file=stdout) + verbose_log(' (binary format check)') cmd = WASM_AS + [wast, '-o', as_file, '-all', '-g'] - verbose_log(' ', ' '.join(cmd), file=stdout) + verbose_log(' ', ' '.join(cmd)) if os.path.exists(as_file): os.unlink(as_file) subprocess.check_call(cmd, stdout=subprocess.PIPE) assert os.path.exists(as_file) cmd = WASM_DIS + [as_file, '-o', disassembled_file, '-all'] - verbose_log(' ', ' '.join(cmd), file=stdout) + verbose_log(' ', ' '.join(cmd)) if os.path.exists(disassembled_file): os.unlink(disassembled_file) subprocess.check_call(cmd, stdout=subprocess.PIPE) @@ -497,7 +497,7 @@ def binary_format_check(wast, verify_final_result=True, base_name=None, stdout=N # make sure it is a valid wast cmd = WASM_OPT + [disassembled_file, '-all', '-q'] - verbose_log(' ', ' '.join(cmd), file=stdout) + verbose_log(' ', ' '.join(cmd)) subprocess.check_call(cmd, stdout=subprocess.PIPE) if verify_final_result: diff --git a/scripts/test/support.py b/scripts/test/support.py index 0cfcfaf7c53..c79bf26ae6c 100644 --- a/scripts/test/support.py +++ b/scripts/test/support.py @@ -16,8 +16,6 @@ import re import subprocess -from . import shared - QUOTED = re.compile(r'\(module\s*(\$\S*)?\s+(quote|binary)') MODULE_DEFINITION_OR_INSTANCE = re.compile(r'(?m)\(module\s+(instance|definition)') @@ -149,7 +147,7 @@ def run_command(cmd, expected_status=0, stdout=None, stderr=None, assert stderr == subprocess.PIPE or stderr is None, \ "Can't redirect stderr if using expected_err" stderr = subprocess.PIPE - shared.verbose_log('executing: ', ' '.join(cmd), file=stdout) + print('executing: ', ' '.join(cmd), file=stdout) out, err, code = _subprocess_run(cmd, stdout=subprocess.PIPE, stderr=stderr, encoding='UTF-8') From 7f5b2c20ccfc4c29b49825ad603375a9fbe7fbd6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 8 May 2026 12:00:14 -0700 Subject: [PATCH 4/5] ruff --- check.py | 4 ++-- scripts/test/shared.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/check.py b/check.py index 5c6b9ef4c90..d54d077993d 100755 --- a/check.py +++ b/check.py @@ -189,7 +189,7 @@ def run_opt_test(wast, stdout=None): support.run_command(cmd, stdout=stdout) -def check_expected(actual, expected, stdout=None): +def check_expected(actual, expected): if expected and os.path.exists(expected): expected = open(expected).read() shared.verbose_log(' (using expected output)') @@ -254,7 +254,7 @@ def run_one_spec_test(wast: Path, stdout=None): support.write_wast(split_name, module) run_opt_test(split_name, stdout=stdout) # also that our optimizer doesn't break on it - result_wast_file = shared.binary_format_check(split_name, verify_final_result=False, base_name=base_name, stdout=stdout) + result_wast_file = shared.binary_format_check(split_name, verify_final_result=False, base_name=base_name) with open(result_wast_file) as f: result_wast = f.read() # add the asserts, and verify that the test still passes diff --git a/scripts/test/shared.py b/scripts/test/shared.py index 492b90b4454..d97bafb98a7 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -474,7 +474,7 @@ def _can_run_spec_test(test): # check utilities -def binary_format_check(wast, verify_final_result=True, base_name=None, stdout=None): +def binary_format_check(wast, verify_final_result=True, base_name=None): # checks we can convert the wast to binary and back as_file = f"{base_name}-a.wasm" if base_name is not None else "a.wasm" From 63e6f5a7b0fc584e24432aa6df0384177ec5b1df Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 8 May 2026 12:53:05 -0700 Subject: [PATCH 5/5] fix --- check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check.py b/check.py index d54d077993d..dddddb424f0 100755 --- a/check.py +++ b/check.py @@ -235,7 +235,7 @@ def run_one_spec_test(wast: Path, stdout=None): shared.fail_with_error(str(e)) raise - check_expected(actual, expected, stdout=stdout) + check_expected(actual, expected) if not is_splittable(wast): return @@ -262,7 +262,7 @@ def run_one_spec_test(wast: Path, stdout=None): # compare all the outputs to the expected output actual = run_spec_test(transformed_path, stdout=stdout) - check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', test_name + '.log'), stdout=stdout) + check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', test_name + '.log')) def run_spec_test_with_wrapped_stdout(wast: Path):