diff --git a/tests/benchmark_utf16.cpp b/tests/benchmark_utf16.cpp index 80f7cbacafeb4..3d152e7b31947 100644 --- a/tests/benchmark_utf16.cpp +++ b/tests/benchmark_utf16.cpp @@ -63,8 +63,5 @@ int main() { } double t3 = emscripten_get_now(); printf("OK. Time: %f (%f).\n", t, t3-t2); - -#ifdef REPORT_RESULT - REPORT_RESULT(0); -#endif + return 0; } diff --git a/tests/benchmark_utf8.cpp b/tests/benchmark_utf8.cpp index 6088feb63439a..6b541fabbe1b0 100644 --- a/tests/benchmark_utf8.cpp +++ b/tests/benchmark_utf8.cpp @@ -64,8 +64,5 @@ int main() { } double t3 = emscripten_get_now(); printf("OK. Time: %f (%f).\n", t, t3-t2); - -#ifdef REPORT_RESULT - REPORT_RESULT(0); -#endif + return 0; } diff --git a/tests/runner.py b/tests/runner.py index 4d127a60d9d51..f5215c26025a2 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -17,8 +17,9 @@ # XXX Use EMTEST_ALL_ENGINES=1 in the env to test all engines! -from subprocess import PIPE, STDOUT +from enum import Enum from functools import wraps +from subprocess import PIPE, STDOUT import argparse import atexit import contextlib @@ -1239,6 +1240,18 @@ def log_request(code=0, size=0): httpd.serve_forever() # test runner will kill us +class Reporting(Enum): + """When running browser tests we normally automatically include support + code for reporting results back to the browser. This enum allows tests + to decide what type of support code they need/want. + """ + NONE = 0 + # Include the JS helpers for reporting results + JS_ONLY = 1 + # Include C/C++ reporting code (REPORT_RESULT mactros) as well as JS helpers + FULL = 2 + + class BrowserCore(RunnerCore): # note how many tests hang / do not send an output. if many of these # happen, likely something is broken and it is best to abort the test @@ -1476,26 +1489,42 @@ def reftest(self, expected, manually_trigger=False): } ''' % (reporting.read(), basename, int(manually_trigger))) - def compile_btest(self, args, reporting=True): - # add in support for reporting results. this adds an include a header so testcases can + def compile_btest(self, args, reporting=Reporting.FULL): + # Inject support code for reporting results. This adds an include a header so testcases can # use REPORT_RESULT, and also adds a cpp file to be compiled alongside the testcase, which # contains the implementation of REPORT_RESULT (we can't just include that implementation in # the header as there may be multiple files being compiled here). args += ['-s', 'IN_TEST_HARNESS=1'] - if reporting: + if reporting != Reporting.NONE: + # For basic reporting we inject JS helper funtions to report result back to server. args += ['-DEMTEST_PORT_NUMBER=%d' % self.port, - '-I', path_from_root('tests'), - '-include', path_from_root('tests', 'report_result.h'), - path_from_root('tests', 'report_result.cpp'), '--pre-js', path_from_root('tests', 'browser_reporting.js')] + if reporting == Reporting.FULL: + # If C reporting (i.e. REPORT_RESULT macro) is required + # also compile in report_result.cpp and forice-include report_result.h + args += ['-I', path_from_root('tests'), + '-include', path_from_root('tests', 'report_result.h'), + path_from_root('tests', 'report_result.cpp')] self.run_process([EMCC] + self.get_emcc_args() + args) + def btest_exit(self, filename, expected, *args, **kwargs): + """Special case of btest that reports its result solely via exiting + with a give result code. + + In this case we set EXIT_RUNTIME and we don't need to provide the + REPORT_RESULT macro to the C code. + """ + self.set_setting('EXIT_RUNTIME') + kwargs['reporting'] = Reporting.JS_ONLY + kwargs['expected'] = 'exit:%s' % expected + return self.btest(filename, *args, **kwargs) + def btest(self, filename, expected=None, reference=None, force_c=False, reference_slack=0, manual_reference=False, post_build=None, args=None, message='.', also_proxied=False, url_suffix='', timeout=None, also_asmjs=False, manually_trigger_reftest=False, extra_tries=1, - reporting=True): + reporting=Reporting.FULL): assert expected or reference, 'a btest must either expect an output, or have a reference image' if args is None: args = [] @@ -1519,7 +1548,7 @@ def btest(self, filename, expected=None, reference=None, force_c=False, args = [filepath, '-o', outfile] + args # print('all args:', args) try_delete(outfile) - self.compile_btest(args, reporting) + self.compile_btest(args, reporting=reporting) self.assertExists(outfile) if post_build: post_build() diff --git a/tests/test_browser.py b/tests/test_browser.py index 025c2a0f40080..92167749e8f26 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -20,7 +20,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from urllib.request import urlopen -from runner import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER +from runner import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER, Reporting from runner import create_test_file, parameterized, ensure_dir from tools import building from tools import shared @@ -1283,7 +1283,7 @@ def test_emscripten_get_now(self): self.btest('emscripten_get_now.cpp', '1', args=args) def test_write_file_in_environment_web(self): - self.btest('write_file.cpp', '0', args=['-s', 'ENVIRONMENT=web', '-Os', '--closure', '1']) + self.btest_exit('write_file.c', 0, args=['-s', 'ENVIRONMENT=web', '-Os', '--closure', '1']) @unittest.skip('Skipping due to https://github.com/emscripten-core/emscripten/issues/2770') def test_fflush(self): @@ -2412,14 +2412,14 @@ def test_runtime_misuse(self): print('mem init, so async, call too early') create_test_file('post.js', post_prep + post_test + post_hook) - self.btest(filename, expected='600', args=['--post-js', 'post.js', '--memory-init-file', '1', '-s', 'EXIT_RUNTIME=1'] + extra_args + mode, reporting=False) + self.btest(filename, expected='600', args=['--post-js', 'post.js', '--memory-init-file', '1', '-s', 'EXIT_RUNTIME=1'] + extra_args + mode, reporting=Reporting.NONE) print('sync startup, call too late') create_test_file('post.js', post_prep + 'Module.postRun.push(function() { ' + post_test + ' });' + post_hook) - self.btest(filename, expected=str(second_code), args=['--post-js', 'post.js', '-s', 'EXIT_RUNTIME=1'] + extra_args + mode, reporting=False) + self.btest(filename, expected=str(second_code), args=['--post-js', 'post.js', '-s', 'EXIT_RUNTIME=1'] + extra_args + mode, reporting=Reporting.NONE) print('sync, runtime still alive, so all good') create_test_file('post.js', post_prep + 'expected_ok = true; Module.postRun.push(function() { ' + post_test + ' });' + post_hook) - self.btest(filename, expected='606', args=['--post-js', 'post.js'] + extra_args + mode, reporting=False) + self.btest(filename, expected='606', args=['--post-js', 'post.js'] + extra_args + mode, reporting=Reporting.NONE) def test_cwrap_early(self): self.btest(os.path.join('browser', 'cwrap_early.cpp'), args=['-O2', '-s', 'ASSERTIONS=1', '--pre-js', path_from_root('tests', 'browser', 'cwrap_early.js'), '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["cwrap"]'], expected='0') @@ -3806,7 +3806,7 @@ def test_pthread_iostream(self): @requires_threads def test_pthread_unistd_io_bigint(self): - self.btest(path_from_root('tests', 'unistd', 'io.c'), expected='0', args=['-s', 'USE_PTHREADS', '-s', 'PROXY_TO_PTHREAD', '-s', 'WASM_BIGINT']) + self.btest_exit(path_from_root('tests', 'unistd', 'io.c'), 0, args=['-s', 'USE_PTHREADS', '-s', 'PROXY_TO_PTHREAD', '-s', 'WASM_BIGINT']) # Test that the main thread is able to use pthread_set/getspecific. @requires_threads @@ -4133,10 +4133,10 @@ def test_wasm_locate_file(self): self.run_browser('test.html', '', '/report_result?0') def test_utf8_textdecoder(self): - self.btest('benchmark_utf8.cpp', expected='0', args=['--embed-file', path_from_root('tests/utf8_corpus.txt') + '@/utf8_corpus.txt', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["UTF8ToString"]']) + self.btest_exit('benchmark_utf8.cpp', 0, args=['--embed-file', path_from_root('tests/utf8_corpus.txt') + '@/utf8_corpus.txt', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["UTF8ToString"]']) def test_utf16_textdecoder(self): - self.btest('benchmark_utf16.cpp', expected='0', args=['--embed-file', path_from_root('tests/utf16_corpus.txt') + '@/utf16_corpus.txt', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["UTF16ToString","stringToUTF16","lengthBytesUTF16"]']) + self.btest_exit('benchmark_utf16.cpp', 0, args=['--embed-file', path_from_root('tests/utf16_corpus.txt') + '@/utf16_corpus.txt', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["UTF16ToString","stringToUTF16","lengthBytesUTF16"]']) def test_TextDecoder(self): self.btest('browser_test_hello_world.c', '0', args=['-s', 'TEXTDECODER=0']) @@ -4443,18 +4443,18 @@ def test_asmfs_dirent_test_readdir_empty(self): @requires_asmfs @requires_threads def test_asmfs_unistd_close(self): - self.btest('unistd/close.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'WASM=0', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1']) + self.btest_exit(path_from_root('tests', 'unistd', 'close.c'), 0, args=['-s', 'ASMFS=1', '-s', 'WASM=0', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1']) @requires_asmfs @requires_threads def test_asmfs_unistd_access(self): - self.btest('unistd/access.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'WASM=0', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1']) + self.btest_exit(path_from_root('tests', 'unistd', 'access.c'), 0, args=['-s', 'ASMFS=1', '-s', 'WASM=0', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1']) @requires_asmfs @requires_threads def test_asmfs_unistd_unlink(self): # TODO: Once symlinks are supported, remove -DNO_SYMLINK=1 - self.btest('unistd/unlink.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'WASM=0', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '-DNO_SYMLINK=1']) + self.btest_exit(path_from_root('tests', 'unistd', 'unlink.c'), 0, args=['-s', 'ASMFS=1', '-s', 'WASM=0', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '-DNO_SYMLINK=1']) @requires_asmfs @requires_threads diff --git a/tests/unistd/access.c b/tests/unistd/access.c index a3691ea4fa045..e8a2d7f13c216 100644 --- a/tests/unistd/access.c +++ b/tests/unistd/access.c @@ -75,9 +75,5 @@ int main() { ); #endif -#ifdef REPORT_RESULT - REPORT_RESULT(0); -#endif - return 0; } diff --git a/tests/unistd/close.c b/tests/unistd/close.c index dcc80add1bbbd..05ecf12d625d9 100644 --- a/tests/unistd/close.c +++ b/tests/unistd/close.c @@ -42,8 +42,5 @@ int main() { assert(errno == EBADF); errno = 0; -#ifdef REPORT_RESULT - REPORT_RESULT(0); -#endif return 0; } diff --git a/tests/unistd/unlink.c b/tests/unistd/unlink.c index eb51f1221a578..d63cca44cd3b3 100644 --- a/tests/unistd/unlink.c +++ b/tests/unistd/unlink.c @@ -187,8 +187,5 @@ int main() { setup(); test(); -#ifdef REPORT_RESULT - REPORT_RESULT(0); -#endif return EXIT_SUCCESS; } diff --git a/tests/write_file.cpp b/tests/write_file.c similarity index 73% rename from tests/write_file.cpp rename to tests/write_file.c index 2bee9b716fdcf..a59f27e350ec2 100644 --- a/tests/write_file.cpp +++ b/tests/write_file.c @@ -9,8 +9,5 @@ int main() { char str[256] = {}; fgets(str, 255, handle); printf("%s\n", str); -#ifdef REPORT_RESULT - REPORT_RESULT(strcmp(str, "hello from file!")); -#endif - return 0; + return strcmp(str, "hello from file!"); }