Optimize JS compiler caching for standard builds (#27026)

Linker system stubs, such as libpthread_stub.js, are dynamically
appended to settings.JS_LIBRARIES for all standard C/C++ builds.
Previously, the JS output cache check did not distinguish between these
system stubs and user-defined JS libraries (passed via --js-library). As
a result, standard builds always bypassed the compilation cache,
triggering a Node.js compilation run on every invocation.

Restrict the cache bypass condition to check only for user libraries
(libraries located outside the Emscripten source tree). Standard
compilations now successfully hit the JS output cache, decreasing
baseline compilation wall clock time of a hello world C program by 52%
(from 472 ms down to 282 ms).

Add an integration test to validate JavaScript compilation caching
(covering hits, misses, user library bypasses, and option-based cache
entries).

Author: brendandahl

test/test_sanity.py

@@ -870,3 +870,45 @@ def test_emcc_with_relative_python_path(self):
     output = self.do([EMCC, test_file('hello_world.c')], env=env)
     self.assertNotContained('error', output)
     self.assertExists('a.out.js')
+
+  def test_emcc_javascript_compilation_caching(self):
+    restore_and_set_up()
+
+    # Create a separate temporary cache folder to avoid dirtying or reading from the default cache.
+    test_cache_dir = self.in_dir('test_cache')
+    js_output_cache_dir = os.path.join(test_cache_dir, 'js_output')
+
+    def js_cache_files():
+      if not os.path.exists(js_output_cache_dir):
+        return []
+      return sorted([f for f in os.listdir(js_output_cache_dir) if f.endswith('.js')])
+
+    def js_cache_size():
+      return len(js_cache_files())
+
+    with env_modify({'EM_CACHE': test_cache_dir}):
+      # 1. First compile. Cache-miss: should compile and populate the cache.
+      self.run_process([EMCC, test_file('hello_world.c'), '-O2', '-o', 'out.js'])
+      self.assertExists(js_output_cache_dir)
+
+      self.assertEqual(js_cache_size(), 1, f'Expected 1 cached JS file, found: {js_cache_files()}')
+
+      cached_file_path = os.path.join(js_output_cache_dir, js_cache_files()[0])
+      initial_mtime = os.path.getmtime(cached_file_path)
+
+      # 2. Second compile. Cache-hit: mtime of cache file should remain strictly identical (not overwritten).
+      self.run_process([EMCC, test_file('hello_world.c'), '-O2', '-o', 'out.js'])
+
+      self.assertEqual(js_cache_size(), 1)
+      self.assertEqual(os.path.getmtime(cached_file_path), initial_mtime, 'Cache was overwritten on second compile (expected a cache hit)')
+
+      # 3. Third compile with custom user library. Cache-bypass: should not add any cache entries.
+      create_file('my_lib.js', 'addToLibrary({ my_custom_symbol: () => {} });')
+      self.run_process([EMCC, test_file('hello_world.c'), '-O2', '--js-library=my_lib.js', '-o', 'out.js'])
+
+      self.assertEqual(js_cache_size(), 1, 'Cache entry was incorrectly created for custom JS library compile')
+
+      # 4. Fourth compile with a changed compiler option. Distinct Cache Entry: should generate a second cache entry.
+      self.run_process([EMCC, test_file('hello_world.c'), '-O2', '-sASSERTIONS=1', '-o', 'out.js'])
+
+      self.assertEqual(js_cache_size(), 2, f'Expected 2 cached JS files after compiling with different options, found: {js_cache_files()}')

tools/emscripten.py

@@ -346,7 +346,8 @@ def compile_javascript_cached():
   # this step is performed while the cache is locked.
   # Sadly we have to skip the caching whenever we have user JS libraries.  This is because
   # these libraries can import arbitrary other JS files (either vis node's `import` or via #include)
-  if DEBUG or settings.BOOTSTRAPPING_STRUCT_INFO or config.FROZEN_CACHE or settings.JS_LIBRARIES:
+  has_user_libs = any(not lib.startswith(utils.path_from_root('src/')) for lib in settings.JS_LIBRARIES)
+  if DEBUG or settings.BOOTSTRAPPING_STRUCT_INFO or config.FROZEN_CACHE or has_user_libs:
     return compile_javascript()
 
   content_hash = generate_js_compiler_input_hash()