Cache and replay stderr for JavaScript compiler

test/test_sanity.py

@@ -912,3 +912,26 @@ def js_cache_size():
       self.run_process([EMCC, test_file('hello_world.c'), '-O2', '-sASSERTIONS=1', '-o', 'out.js'])
 
       self.assertEqual(js_cache_size(), 2, f'Expected 2 cached JS files after compiling with different options, found: {js_cache_files()}')
+
+  def test_emcc_javascript_compilation_caching_warnings(self):
+    restore_and_set_up()
+
+    # Create a separate temporary cache folder to avoid dirtying or reading from the default cache.
+    test_cache_dir = self.in_dir('test_cache_warn')
+
+    create_file('main.c', r'''
+      void something();
+      int main() {
+        something();
+        return 0;
+      }
+      ''')
+
+    with env_modify({'EM_CACHE': test_cache_dir}):
+      # 1. First compile. Cache-miss: should compile, populate cache with .js and .stderr, and output warning.
+      proc = self.run_process([EMCC, 'main.c', '-sERROR_ON_UNDEFINED_SYMBOLS=0', '-o', 'out.js'], stderr=PIPE)
+      self.assertContained('warning: undefined symbol: something', proc.stderr)
+
+      # 2. Second compile. Cache-hit: should replay the warning from cache.
+      proc = self.run_process([EMCC, 'main.c', '-sERROR_ON_UNDEFINED_SYMBOLS=0', '-o', 'out.js'], stderr=PIPE)
+      self.assertContained('warning: undefined symbol: something', proc.stderr)

tools/emscripten.py

@@ -209,12 +209,6 @@ def generate_js_compiler_input_hash(symbols_only=False):
 
 @ToolchainProfiler.profile()
 def compile_javascript(symbols_only=False):
-  stderr_file = os.environ.get('EMCC_STDERR_FILE')
-  if stderr_file:
-    stderr_file = os.path.abspath(stderr_file)
-    logger.info('logging stderr in js compiler phase into %s' % stderr_file)
-    stderr_file = open(stderr_file, 'w', encoding='utf-8')
-
   # Save settings to a file to work around v8 issue 1579
   settings_json = json.dumps(settings.external_dict(), sort_keys=True, indent=2)
   building.write_intermediate(settings_json, 'settings.json')
@@ -223,8 +217,9 @@ def compile_javascript(symbols_only=False):
   args = ['-']
   if symbols_only:
     args += ['--symbols-only']
-  return shared.run_js_tool(path_from_root('tools/compiler.mjs'),
-                            args, input=settings_json, stdout=subprocess.PIPE, stderr=stderr_file)
+  proc = shared.run_js_tool(path_from_root('tools/compiler.mjs'),
+                            args, input=settings_json, stdout=subprocess.PIPE, stderr=subprocess.PIPE, return_proc=True)
+  return proc.stdout, proc.stderr
 
 
 def set_memory(static_bump):
@@ -302,13 +297,30 @@ def trim_asm_const_body(body):
   return body
 
 
+def output_stderr(stderr):
+  if stderr is not None:
+    stderr_file = os.environ.get('EMCC_STDERR_FILE')
+    if stderr_file:
+      stderr_file = os.path.abspath(stderr_file)
+      logger.info('logging stderr in js compiler phase into %s' % stderr_file)
+      with open(stderr_file, 'w', encoding='utf-8') as f:
+        f.write(stderr)
+    elif stderr:
+      sys.stderr.write(stderr)
+      sys.stderr.flush()
+
+
 def get_cached_file(filetype, filename, generator, cache_limit):
   """Implement a file cache which lives inside the main emscripten cache directory.
 
   The defference here is that we use a per-file lock rather than a cache-wide lock.
 
   The cache is pruned (by removing the oldest files) if it grows above
   a certain number of files.
+
+  The generator must return a tuple of (content, stderr).
+  The stderr output will be cached alongside the main file in a companion .stderr file
+  and successfully replayed on both cache hits and misses.
   """
   root = cache.get_path(filetype)
   utils.safe_ensure_dirs(root)
@@ -317,25 +329,33 @@ def get_cached_file(filetype, filename, generator, cache_limit):
 
   with filelock.FileLock(cache_file + '.lock'):
     if os.path.exists(cache_file):
-      # Cache hit, read the file
+      # Cache hit, read the file and any associated stderr output
       file_content = utils.read_file(cache_file)
+      stderr_content = utils.read_file(cache_file + '.stderr') if os.path.exists(cache_file + '.stderr') else ''
     else:
-      # Cache miss, generate the symbol list and write the file
-      file_content = generator()
+      # Cache miss, generate the content and stderr, and write to cache
+      file_content, stderr_content = generator()
       utils.write_file(cache_file, file_content)
+      if stderr_content:
+        utils.write_file(cache_file + '.stderr', stderr_content)
+
+    # Replay cached stderr output to the appropriate stream/file
+    output_stderr(stderr_content)
 
-  if len([f for f in os.listdir(root) if not f.endswith('.lock')]) > cache_limit:
+  # Exclude .stderr companion files from the cache limit count, as they are managed alongside their primary files
+  if len([f for f in os.listdir(root) if not f.endswith(('.lock', '.stderr'))]) > cache_limit:
     with filelock.FileLock(cache.get_path(f'{filetype}.lock')):
       files = []
       for f in os.listdir(root):
-        if not f.endswith('.lock'):
+        if not f.endswith(('.lock', '.stderr')):
           f = os.path.join(root, f)
           files.append((f, os.path.getmtime(f)))
       files.sort(key=lambda x: x[1])
-      # Delete all but the newest N files
+      # Delete all but the newest N files (and remove their companion .stderr files if present)
       for f, _ in files[:-cache_limit]:
         with filelock.FileLock(f + '.lock'):
           utils.delete_file(f)
+          utils.delete_file(f + '.stderr')
 
   return file_content
 
@@ -348,7 +368,9 @@ def compile_javascript_cached():
   # these libraries can import arbitrary other JS files (either vis node's `import` or via #include)
   has_user_libs = any(not lib.startswith(utils.path_from_root('src/')) for lib in settings.JS_LIBRARIES)
   if DEBUG or settings.BOOTSTRAPPING_STRUCT_INFO or config.FROZEN_CACHE or has_user_libs:
-    return compile_javascript()
+    out, stderr = compile_javascript()
+    output_stderr(stderr)
+    return out
 
   content_hash = generate_js_compiler_input_hash()
 

tools/link.py

@@ -220,23 +220,25 @@ def generate_js_sym_info():
   mode of the JS compiler that would generate a list of all possible symbols
   that could be checked in.
   """
-  output = emscripten.compile_javascript(symbols_only=True)
+  output, stderr = emscripten.compile_javascript(symbols_only=True)
   # When running in symbols_only mode compiler.mjs outputs symbol metadata as JSON.
-  return json.loads(output)
+  return json.loads(output), stderr
 
 
 @ToolchainProfiler.profile_block('JS symbol generation')
 def get_js_sym_info():
   # Avoiding using the cache when generating struct info since
   # this step is performed while the cache is locked.
   if DEBUG or settings.BOOTSTRAPPING_STRUCT_INFO or config.FROZEN_CACHE:
-    return generate_js_sym_info()
+    syms, stderr = generate_js_sym_info()
+    emscripten.output_stderr(stderr)
+    return syms
 
   content_hash = emscripten.generate_js_compiler_input_hash(symbols_only=True)
 
   def generate_json():
-    library_syms = generate_js_sym_info()
-    return json.dumps(library_syms, separators=(',', ':'), indent=2)
+    library_syms, stderr = generate_js_sym_info()
+    return json.dumps(library_syms, separators=(',', ':'), indent=2), stderr
 
   # Limit of the overall size of the cache.
   # This code will get test coverage since a full test run of `other` or `core`

tools/shared.py

@@ -201,14 +201,17 @@ def exec_process(cmd):
   utils.exec(cmd)
 
 
-def run_js_tool(filename, jsargs=[], node_args=[], **kw):  # noqa: B006
+def run_js_tool(filename, jsargs=[], node_args=[], return_proc=False, **kw):  # noqa: B006
   """Execute a javascript tool.
 
   This is used by emcc to run parts of the build process that are
   implemented in javascript.
   """
   command = [*config.NODE_JS, *node_args, filename, *jsargs]
-  return check_call(command, **kw).stdout
+  proc = check_call(command, **kw)
+  if return_proc:
+    return proc
+  return proc.stdout
 
 
 def get_npm_cmd(name, missing_ok=False):