diff --git a/test/test_sanity.py b/test/test_sanity.py index add24e67d2be7..3dda2a47224d4 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -912,3 +912,26 @@ def js_cache_size(): self.run_process([EMCC, test_file('hello_world.c'), '-O2', '-sASSERTIONS=1', '-o', 'out.js']) self.assertEqual(js_cache_size(), 2, f'Expected 2 cached JS files after compiling with different options, found: {js_cache_files()}') + + def test_emcc_javascript_compilation_caching_warnings(self): + restore_and_set_up() + + # Create a separate temporary cache folder to avoid dirtying or reading from the default cache. + test_cache_dir = self.in_dir('test_cache_warn') + + create_file('main.c', r''' + void something(); + int main() { + something(); + return 0; + } + ''') + + with env_modify({'EM_CACHE': test_cache_dir}): + # 1. First compile. Cache-miss: should compile, populate cache with .js and .stderr, and output warning. + proc = self.run_process([EMCC, 'main.c', '-sERROR_ON_UNDEFINED_SYMBOLS=0', '-o', 'out.js'], stderr=PIPE) + self.assertContained('warning: undefined symbol: something', proc.stderr) + + # 2. Second compile. Cache-hit: should replay the warning from cache. + proc = self.run_process([EMCC, 'main.c', '-sERROR_ON_UNDEFINED_SYMBOLS=0', '-o', 'out.js'], stderr=PIPE) + self.assertContained('warning: undefined symbol: something', proc.stderr) diff --git a/tools/emscripten.py b/tools/emscripten.py index fe51b70ea19d6..19be9284e949c 100644 --- a/tools/emscripten.py +++ b/tools/emscripten.py @@ -209,12 +209,6 @@ def generate_js_compiler_input_hash(symbols_only=False): @ToolchainProfiler.profile() def compile_javascript(symbols_only=False): - stderr_file = os.environ.get('EMCC_STDERR_FILE') - if stderr_file: - stderr_file = os.path.abspath(stderr_file) - logger.info('logging stderr in js compiler phase into %s' % stderr_file) - stderr_file = open(stderr_file, 'w', encoding='utf-8') - # Save settings to a file to work around v8 issue 1579 settings_json = json.dumps(settings.external_dict(), sort_keys=True, indent=2) building.write_intermediate(settings_json, 'settings.json') @@ -223,8 +217,9 @@ def compile_javascript(symbols_only=False): args = ['-'] if symbols_only: args += ['--symbols-only'] - return shared.run_js_tool(path_from_root('tools/compiler.mjs'), - args, input=settings_json, stdout=subprocess.PIPE, stderr=stderr_file) + proc = shared.run_js_tool(path_from_root('tools/compiler.mjs'), + args, input=settings_json, stdout=subprocess.PIPE, stderr=subprocess.PIPE, return_proc=True) + return proc.stdout, proc.stderr def set_memory(static_bump): @@ -302,6 +297,19 @@ def trim_asm_const_body(body): return body +def output_stderr(stderr): + if stderr is not None: + stderr_file = os.environ.get('EMCC_STDERR_FILE') + if stderr_file: + stderr_file = os.path.abspath(stderr_file) + logger.info('logging stderr in js compiler phase into %s' % stderr_file) + with open(stderr_file, 'w', encoding='utf-8') as f: + f.write(stderr) + elif stderr: + sys.stderr.write(stderr) + sys.stderr.flush() + + def get_cached_file(filetype, filename, generator, cache_limit): """Implement a file cache which lives inside the main emscripten cache directory. @@ -309,6 +317,10 @@ def get_cached_file(filetype, filename, generator, cache_limit): The cache is pruned (by removing the oldest files) if it grows above a certain number of files. + + The generator must return a tuple of (content, stderr). + The stderr output will be cached alongside the main file in a companion .stderr file + and successfully replayed on both cache hits and misses. """ root = cache.get_path(filetype) utils.safe_ensure_dirs(root) @@ -317,25 +329,33 @@ def get_cached_file(filetype, filename, generator, cache_limit): with filelock.FileLock(cache_file + '.lock'): if os.path.exists(cache_file): - # Cache hit, read the file + # Cache hit, read the file and any associated stderr output file_content = utils.read_file(cache_file) + stderr_content = utils.read_file(cache_file + '.stderr') if os.path.exists(cache_file + '.stderr') else '' else: - # Cache miss, generate the symbol list and write the file - file_content = generator() + # Cache miss, generate the content and stderr, and write to cache + file_content, stderr_content = generator() utils.write_file(cache_file, file_content) + if stderr_content: + utils.write_file(cache_file + '.stderr', stderr_content) + + # Replay cached stderr output to the appropriate stream/file + output_stderr(stderr_content) - if len([f for f in os.listdir(root) if not f.endswith('.lock')]) > cache_limit: + # Exclude .stderr companion files from the cache limit count, as they are managed alongside their primary files + if len([f for f in os.listdir(root) if not f.endswith(('.lock', '.stderr'))]) > cache_limit: with filelock.FileLock(cache.get_path(f'{filetype}.lock')): files = [] for f in os.listdir(root): - if not f.endswith('.lock'): + if not f.endswith(('.lock', '.stderr')): f = os.path.join(root, f) files.append((f, os.path.getmtime(f))) files.sort(key=lambda x: x[1]) - # Delete all but the newest N files + # Delete all but the newest N files (and remove their companion .stderr files if present) for f, _ in files[:-cache_limit]: with filelock.FileLock(f + '.lock'): utils.delete_file(f) + utils.delete_file(f + '.stderr') return file_content @@ -348,7 +368,9 @@ def compile_javascript_cached(): # these libraries can import arbitrary other JS files (either vis node's `import` or via #include) has_user_libs = any(not lib.startswith(utils.path_from_root('src/')) for lib in settings.JS_LIBRARIES) if DEBUG or settings.BOOTSTRAPPING_STRUCT_INFO or config.FROZEN_CACHE or has_user_libs: - return compile_javascript() + out, stderr = compile_javascript() + output_stderr(stderr) + return out content_hash = generate_js_compiler_input_hash() diff --git a/tools/link.py b/tools/link.py index 2065b9d3d6f9b..e66754985bb99 100644 --- a/tools/link.py +++ b/tools/link.py @@ -220,9 +220,9 @@ def generate_js_sym_info(): mode of the JS compiler that would generate a list of all possible symbols that could be checked in. """ - output = emscripten.compile_javascript(symbols_only=True) + output, stderr = emscripten.compile_javascript(symbols_only=True) # When running in symbols_only mode compiler.mjs outputs symbol metadata as JSON. - return json.loads(output) + return json.loads(output), stderr @ToolchainProfiler.profile_block('JS symbol generation') @@ -230,13 +230,15 @@ def get_js_sym_info(): # Avoiding using the cache when generating struct info since # this step is performed while the cache is locked. if DEBUG or settings.BOOTSTRAPPING_STRUCT_INFO or config.FROZEN_CACHE: - return generate_js_sym_info() + syms, stderr = generate_js_sym_info() + emscripten.output_stderr(stderr) + return syms content_hash = emscripten.generate_js_compiler_input_hash(symbols_only=True) def generate_json(): - library_syms = generate_js_sym_info() - return json.dumps(library_syms, separators=(',', ':'), indent=2) + library_syms, stderr = generate_js_sym_info() + return json.dumps(library_syms, separators=(',', ':'), indent=2), stderr # Limit of the overall size of the cache. # This code will get test coverage since a full test run of `other` or `core` diff --git a/tools/shared.py b/tools/shared.py index d03cab7dd2bd5..de2ebbde465a2 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -201,14 +201,17 @@ def exec_process(cmd): utils.exec(cmd) -def run_js_tool(filename, jsargs=[], node_args=[], **kw): # noqa: B006 +def run_js_tool(filename, jsargs=[], node_args=[], return_proc=False, **kw): # noqa: B006 """Execute a javascript tool. This is used by emcc to run parts of the build process that are implemented in javascript. """ command = [*config.NODE_JS, *node_args, filename, *jsargs] - return check_call(command, **kw).stdout + proc = check_call(command, **kw) + if return_proc: + return proc + return proc.stdout def get_npm_cmd(name, missing_ok=False):