Merge branch 'main' into simplify-thread-exit

Author: kleisauke

.circleci/config.yml

@@ -1095,6 +1095,7 @@ jobs:
             core0.test_async_ccall_promise_jspi*
             core0.test_cubescript_jspi
             core0.test_poll_blocking_asyncify_jspi
+            wasm64.test_pthread_join_and_asyncify
             "
       - upload-test-results
   test-other:

ChangeLog.md

@@ -20,6 +20,11 @@ See docs/process.md for more on how version tagging works.
 
 5.0.8 (in development)
 ----------------------
+- When performing a streaming Fetch operation, the max chunk size of downloaded
+  bytes that is handed over to the Wasm side from JS is now capped to maximum
+  of 8 megabytes. This ensures that a streaming Fetch stays streaming, rather
+  than transferring the whole (potentially large) file as one huge chunk, which
+  might not fit in the WebAssembly memory. (#26898)
 - The minimum versions of browser engines supported by emscripten's generated
   code were bumped, allowing us to remove our internal support for transpilation
   via babel:
@@ -31,6 +36,11 @@ See docs/process.md for more on how version tagging works.
   emscripten.  If you still need to support extremely old browsers, you can
   manually transpile the output of emscripten (e.g. using babel for JS and
   binaryen for wasm). (#26677)
+- The `-m64` compiler flag is now honored, and works as an alias for
+  `-sMEMORY64` and/or `--target=wasm64`. (#26765)
+- The autopersistence feature in IDBFS mount now supports registering a global
+  callback `IDBFS.onAutoPersistStateChanged = active => {}`, which will be
+  notified of all IDBFS sync start and end events. (#26895)
 
 5.0.7 - 04/30/26
 ----------------
@@ -54,8 +64,6 @@ See docs/process.md for more on how version tagging works.
   a Wasm Worker.  This mode increases the memory used by each Wasm Worker by
   ~500 bytes (in the same way that declaring ~500 bytes of TLS data would).
   (#26757)
-- The `-m64` compiler flag is now honored, and works are an alias for
-  `-sMEMORY64` and/or `--target=wasm64`. (#26765)
 - The filesystem opteration that create new files now honor the global umask,
   which defaults for 0o222 and can be updated by calling `umask()`. (#50739)
 

src/Fetch.js

@@ -661,30 +661,47 @@ function fetchXHR(fetch, onsuccess, onerror, onprogress, onreadystatechange) {
       return;
     }
     var ptrLen = (fetchAttrLoadToMemory && fetchAttrStreamData) ? xhr.response?.byteLength ?? 0 : 0;
-    var ptr = 0;
-    if (ptrLen > 0 && fetchAttrLoadToMemory && fetchAttrStreamData) {
+
+    // Specifies the maximum chunk size that a streaming fetch will transfer from
+    // JS over to WebAssembly side. Used to cap a streaming fetch to avoid
+    // overallocating WebAssembly memory needlessly.
+    var FETCH_STREAMING_MAX_CHUNK_SIZE = 8*1024*1024;
+
+    for (var bytePos = 0; bytePos < ptrLen || !ptrLen;) {
+      var sz = Math.min(ptrLen - bytePos, FETCH_STREAMING_MAX_CHUNK_SIZE);
+
+      var ptr = 0;
+      if (sz > 0 && fetchAttrLoadToMemory && fetchAttrStreamData) {
+        // Even though we are doing a streaming fetch (i.e. in small chunks), Safari may call onprogress with a huge
+        // chunk size. This will be a problem for Wasm applications that intend to use streaming fetch to process
+        // an input file in small chunks (to avoid blowing up the WebAssembly heap size). Therefore apply a max
+        // chunk size ceiling to the received chunks, and transfer the data over to WebAssembly using max sized chunks.
+
 #if FETCH_DEBUG
-      dbg(`fetch: allocating ${ptrLen} bytes in Emscripten heap for xhr data`);
+        dbg(`fetch: allocating ${sz} bytes in Emscripten heap for xhr data`);
 #endif
 #if ASSERTIONS
-      assert(onprogress, 'streaming fetch requires an onprogress handler');
+        assert(onprogress, 'streaming fetch requires an onprogress handler');
 #endif
-      // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is
-      // freed when emscripten_fetch_close() is called.
-      ptr = _realloc({{{ makeGetValue('fetch', C_STRUCTS.emscripten_fetch_t.data, '*') }}}, ptrLen);
-      HEAPU8.set(new Uint8Array(/** @type{Array<number>} */(xhr.response)), ptr);
+        // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is
+        // freed when emscripten_fetch_close() is called.
+        ptr = _realloc({{{ makeGetValue('fetch', C_STRUCTS.emscripten_fetch_t.data, '*') }}}, sz);
+        HEAPU8.set(new Uint8Array(/** @type{Array<number>} */(xhr.response), bytePos, sz), ptr);
+      }
+      {{{ makeSetValue('fetch', C_STRUCTS.emscripten_fetch_t.data, 'ptr', '*') }}}
+      writeI53ToI64(fetch + {{{ C_STRUCTS.emscripten_fetch_t.numBytes }}}, sz);
+      writeI53ToI64(fetch + {{{ C_STRUCTS.emscripten_fetch_t.dataOffset }}}, e.loaded - ptrLen + bytePos);
+      writeI53ToI64(fetch + {{{ C_STRUCTS.emscripten_fetch_t.totalBytes }}}, e.total);
+      {{{ makeSetValue('fetch', C_STRUCTS.emscripten_fetch_t.readyState, 'xhr.readyState', 'i16') }}}
+      var status = xhr.status;
+      // If loading files from a source that does not give HTTP status code, assume success if we get data bytes
+      if (xhr.readyState >= 3 && xhr.status === 0 && e.loaded > 0) status = 200;
+      {{{ makeSetValue('fetch', C_STRUCTS.emscripten_fetch_t.status, 'status', 'i16') }}}
+      if (xhr.statusText) stringToUTF8(xhr.statusText, fetch + {{{ C_STRUCTS.emscripten_fetch_t.statusText }}}, 64);
+      onprogress(fetch, e);
+      bytePos += sz;
+      if (!ptrLen) break;
     }
-    {{{ makeSetValue('fetch', C_STRUCTS.emscripten_fetch_t.data, 'ptr', '*') }}}
-    writeI53ToI64(fetch + {{{ C_STRUCTS.emscripten_fetch_t.numBytes }}}, ptrLen);
-    writeI53ToI64(fetch + {{{ C_STRUCTS.emscripten_fetch_t.dataOffset }}}, e.loaded - ptrLen);
-    writeI53ToI64(fetch + {{{ C_STRUCTS.emscripten_fetch_t.totalBytes }}}, e.total);
-    {{{ makeSetValue('fetch', C_STRUCTS.emscripten_fetch_t.readyState, 'xhr.readyState', 'i16') }}}
-    var status = xhr.status;
-    // If loading files from a source that does not give HTTP status code, assume success if we get data bytes
-    if (xhr.readyState >= 3 && xhr.status === 0 && e.loaded > 0) status = 200;
-    {{{ makeSetValue('fetch', C_STRUCTS.emscripten_fetch_t.status, 'status', 'i16') }}}
-    if (xhr.statusText) stringToUTF8(xhr.statusText, fetch + {{{ C_STRUCTS.emscripten_fetch_t.statusText }}}, 64);
-    onprogress(fetch, e);
   };
   xhr.onreadystatechange = (e) => {
     // check if xhr was aborted by user and don't try to call back

src/lib/libglemu.js

@@ -857,7 +857,7 @@ var LibraryGLEmulation = {
 #if !FULL_ES2
   $GLImmediate__postset: 'GLImmediate.setupFuncs(); Browser.moduleContextCreatedCallbacks.push(() => GLImmediate.init());',
 #endif
-  $GLImmediate__deps: ['$Browser', '$GL', '$GLEmulation'],
+  $GLImmediate__deps: ['$Browser', '$GL', '$GLEmulation', '$webglBufferSubData'],
   $GLImmediate: {
     MapTreeLib: null,
     spawnMapTreeLib: () => {
@@ -2552,7 +2552,7 @@ var LibraryGLEmulation = {
               GLImmediate.lastArrayBuffer = arrayBuffer;
             }
 
-            GLctx.bufferSubData(GLctx.ARRAY_BUFFER, start, GLImmediate.vertexData.subarray(start >> 2, end >> 2));
+            webglBufferSubData(GLctx.ARRAY_BUFFER, start, (end - start) >> 2, start >> 2, GLImmediate.vertexData);
           }
 #if GL_UNSAFE_OPTS
           if (canSkip) return;
@@ -3046,12 +3046,13 @@ var LibraryGLEmulation = {
         }
         if (!GLctx.currentElementArrayBufferBinding) {
           // If no element array buffer is bound, then indices is a literal pointer to clientside data
+          var byteSize = numProvidedIndexes << 1;
 #if ASSERTIONS
-          assert(numProvidedIndexes << 1 <= GL.MAX_TEMP_BUFFER_SIZE, 'too many immediate mode indexes (a)');
+          assert(byteSize <= GL.MAX_TEMP_BUFFER_SIZE, 'too many immediate mode indexes (a)');
 #endif
-          var indexBuffer = GL.getTempIndexBuffer(numProvidedIndexes << 1);
+          var indexBuffer = GL.getTempIndexBuffer(byteSize);
           GLctx.bindBuffer(GLctx.ELEMENT_ARRAY_BUFFER, indexBuffer);
-          GLctx.bufferSubData(GLctx.ELEMENT_ARRAY_BUFFER, 0, {{{ makeHEAPView('U16', 'ptr', 'ptr + (numProvidedIndexes << 1)') }}});
+          webglBufferSubData(GLctx.ELEMENT_ARRAY_BUFFER, 0, byteSize, ptr);
           ptr = 0;
           emulatedElementArrayBuffer = true;
         }

src/lib/libidbfs.js

@@ -18,14 +18,31 @@ addToLibrary({
     DB_VERSION: 21,
     DB_STORE_NAME: 'FILE_DATA',
 
+    // When using the autopersistence mechanism, users can set
+    // IDBFS.onAutoPersistStateChanged callback to receive notification events
+    // for when persistence operations are in-flight. Use the following syntax:
+    /*
+    IDBFS.onAutoPersistStateChanged = autoPersistActive => {
+      if (autoPersistActive) {
+        console.log('IDBFS persistence operation has started.');
+      } else {
+        console.log('IDBFS persistence operation has finished.');
+      }
+    };
+    */
+
     // Queues a new VFS -> IDBFS synchronization operation
     queuePersist: (mount) => {
       function onPersistComplete() {
         if (mount.idbPersistState === 'again') startPersist(); // If a new sync request has appeared in between, kick off a new sync
-        else mount.idbPersistState = 0; // Otherwise reset sync state back to idle to wait for a new sync later
+        else {
+          mount.idbPersistState = 0; // Otherwise reset sync state back to idle to wait for a new sync later
+          IDBFS.onAutoPersistStateChanged?.(false);
+        }
       }
       function startPersist() {
         mount.idbPersistState = 'idb'; // Mark that we are currently running a sync operation
+        IDBFS.onAutoPersistStateChanged?.(true);
         IDBFS.syncfs(mount, /*populate:*/false, onPersistComplete);
       }
 

src/lib/libwebgl.js

@@ -249,6 +249,7 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
 #endif // GL_SUPPORT_AUTOMATIC_ENABLE_EXTENSIONS
 #if FULL_ES2 || LEGACY_GL_EMULATION
     '$registerPreMainLoop',
+    '$webglBufferSubData',
 #endif
   ],
 #if FULL_ES2 || LEGACY_GL_EMULATION
@@ -584,9 +585,7 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
         var size = GL.calcBufLength(cb.size, cb.type, cb.stride, count);
         var buf = GL.getTempVertexBuffer(size);
         GLctx.bindBuffer(0x8892 /*GL_ARRAY_BUFFER*/, buf);
-        GLctx.bufferSubData(0x8892 /*GL_ARRAY_BUFFER*/,
-                                 0,
-                                 HEAPU8.subarray(cb.ptr, cb.ptr + size));
+        webglBufferSubData(0x8892 /*GL_ARRAY_BUFFER*/, 0, size, cb.ptr);
 #if GL_ASSERTIONS
         GL.validateVertexAttribPointer(cb.size, cb.type, cb.stride, 0);
 #endif
@@ -1258,6 +1257,23 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
 
   },
 
+  // Wrapper around GLctx.bufferSubData that can hangle both WebGL1 (which
+  // requires new subarray on each call) and WebGL2 (which does not).
+  // Argument ordering is a little strange here, since we want a default
+  // for `src` is has to come last.
+  $webglBufferSubData__internal: true,
+  $webglBufferSubData: (target, offset, size, data, src = HEAPU8) => {
+#if WEBGL_USE_GARBAGE_FREE_APIS
+    if ({{{ isCurrentContextWebGL2() }}}) {
+      size && GLctx.bufferSubData(target, offset, src, data, size);
+      return;
+    }
+#endif
+#if INCLUDE_WEBGL1_FALLBACK
+    GLctx.bufferSubData(target, offset, src.subarray(data, data + size));
+#endif
+  },
+
   $webglGetExtensions__internal: true,
   $webglGetExtensions__deps: ['$getEmscriptenSupportedExtensions'],
   $webglGetExtensions: () => {
@@ -1915,17 +1931,10 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
 #endif
   },
 
-  glBufferSubData: (target, offset, size, data) => {
-#if WEBGL_USE_GARBAGE_FREE_APIS
-    if ({{{ isCurrentContextWebGL2() }}}) {
-      size && GLctx.bufferSubData(target, offset, HEAPU8, data, size);
-      return;
-    }
-#endif
-#if INCLUDE_WEBGL1_FALLBACK
-    GLctx.bufferSubData(target, offset, HEAPU8.subarray(data, data+size));
-#endif
-  },
+  // This cannot be simple alias because under wasm64 we need to be able modify
+  // the function at compile time to provide automatically marshal of the pointer arguments.
+  glBufferSubData__deps: ['$webglBufferSubData'],
+  glBufferSubData: (target, offset, size, data) => webglBufferSubData(target, offset, size, data),
 
   // Queries EXT
   glGenQueriesEXT__sig: 'vip',
@@ -3874,6 +3883,7 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
 #endif
   },
 
+  glDrawElements__deps: ['$webglBufferSubData'],
   glDrawElements: (mode, count, type, indices) => {
 #if FULL_ES2
     var buf;
@@ -3882,9 +3892,7 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
       var size = GL.calcBufLength(1, type, 0, count);
       buf = GL.getTempIndexBuffer(size);
       GLctx.bindBuffer(0x8893 /*GL_ELEMENT_ARRAY_BUFFER*/, buf);
-      GLctx.bufferSubData(0x8893 /*GL_ELEMENT_ARRAY_BUFFER*/,
-                          0,
-                          HEAPU8.subarray(indices, indices + size));
+      webglBufferSubData(0x8893 /*GL_ELEMENT_ARRAY_BUFFER*/, 0, size, indices);
 
       // Calculating vertex count if shader's attribute data is on client side
       if (count > 0) {
@@ -4210,7 +4218,7 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
     }
   },
 
-  glFlushMappedBufferRange__deps: ['$emscriptenWebGLGetBufferBinding', '$emscriptenWebGLValidateMapBufferTarget'],
+  glFlushMappedBufferRange__deps: ['$emscriptenWebGLGetBufferBinding', '$emscriptenWebGLValidateMapBufferTarget', '$webglBufferSubData'],
   glFlushMappedBufferRange: (target, offset, length) => {
     if (!emscriptenWebGLValidateMapBufferTarget(target)) {
       GL.recordError(0x500/*GL_INVALID_ENUM*/);
@@ -4236,10 +4244,7 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
       return;
     }
 
-    GLctx.bufferSubData(
-      target,
-      mapping.offset,
-      HEAPU8.subarray(mapping.mem + offset, mapping.mem + offset + length));
+    webglBufferSubData(target, mapping.offset, length, mapping.mem + offset);
   },
 
   glUnmapBuffer__deps: ['$emscriptenWebGLGetBufferBinding', '$emscriptenWebGLValidateMapBufferTarget', 'free'],
@@ -4259,18 +4264,9 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}};
     }
 
     if (!(mapping.access & 0x10)) { /* GL_MAP_FLUSH_EXPLICIT_BIT */
-#if WEBGL_USE_GARBAGE_FREE_APIS
-      if ({{{ isCurrentContextWebGL2() }}}) {
-        GLctx.bufferSubData(target, mapping.offset, HEAPU8, mapping.mem, mapping.length);
-      }
-#if INCLUDE_WEBGL1_FALLBACK
-      else
-#endif
-#endif
-#if INCLUDE_WEBGL1_FALLBACK
-      GLctx.bufferSubData(target, mapping.offset, HEAPU8.subarray(mapping.mem, mapping.mem+mapping.length));
-#endif
+      webglBufferSubData(target, mapping.offset, mapping.length, mapping.mem);
     }
+
     _free(mapping.mem);
     mapping.mem = 0;
     return 1;

src/parseTools.mjs

@@ -682,6 +682,7 @@ function makeDynCall(sig, funcPtr, promising = false) {
   }
   args = args.join(', ');
 
+  const needRtnConversion = MEMORY64 && sig[0] == 'p';
   const needArgConversion = MEMORY64 && sig.includes('p');
   let callArgs = args;
   if (needArgConversion) {
@@ -751,7 +752,15 @@ Please update to new syntax.`);
   }
 
   if (needArgConversion) {
-    return `((${args}) => ${getWasmTableEntry}.call(null, ${callArgs}))`;
+    if (needRtnConversion) {
+      if (promising) {
+        return `((${args}) => ${getWasmTableEntry}.call(null, ${callArgs}).then(Number))`;
+      } else {
+        return `((${args}) => Number(${getWasmTableEntry}.call(null, ${callArgs})))`;
+      }
+    } else {
+      return `((${args}) => ${getWasmTableEntry}.call(null, ${callArgs}))`;
+    }
   }
   return getWasmTableEntry;
 }

test/browser_harness.html

@@ -23,32 +23,32 @@
     }
   </style>
   <script>
-    var COMMAND_PREFIX = 'COMMAND:';
     var counter = 0;
-    function check() {
-      fetch('/check')
-        .then((rsp) => rsp.text())
-        .then((responseText) => {
-          if (responseText.startsWith(COMMAND_PREFIX)) {
-            var url = responseText.substr(COMMAND_PREFIX.length);
-            iframe.src = url;
-            document.getElementById('count').textContent = counter++;
-          }
-          // Polling for the next test to start: we just keep asking if we
-          // should load a new page, since the iframe doesn't currently have
-          // a way to tell us it completed (even if it did, we'd need to poll
-          // the server to know when the next test page is ready to load).
-          // (A timeout value of 100msecs has been seen to stall test
-          //  browser.test_audio_worklet_worker on Chrome)
-          setTimeout(check, 200);
-        })
-        .catch(() => {
-          document.body.innerHTML = 'Tests complete. View log in console.';
-          // Attempt to close the main window.  This works on chrome
-          // but fails on firefox with: `Scripts may not close windows that
-          // were not opened by script.`
-          window.close();
-        });
+    async function check() {
+      try {
+        const rsp = await fetch('/check');
+        const responseText = await rsp.text();
+
+        const COMMAND_PREFIX = 'COMMAND:';
+        if (responseText.startsWith(COMMAND_PREFIX)) {
+          var url = responseText.substr(COMMAND_PREFIX.length);
+          iframe.src = url;
+          document.getElementById('count').textContent = counter++;
+        }
+        // Polling for the next test to start: we just keep asking if we
+        // should load a new page, since the iframe doesn't currently have
+        // a way to tell us it completed (even if it did, we'd need to poll
+        // the server to know when the next test page is ready to load).
+        // (A timeout value of 100msecs has been seen to stall test
+        //  browser.test_audio_worklet_worker on Chrome)
+        setTimeout(check, 200);
+      } catch {
+        document.body.innerHTML = 'Tests complete. View log in console.';
+        // Attempt to close the main window.  This works on chrome
+        // but fails on firefox with: `Scripts may not close windows that
+        // were not opened by script.`
+        window.close();
+      }
     }
     document.addEventListener('DOMContentLoaded', check);
   </script>

test/codesize/test_codesize_hello_dylink_all.json

@@ -1,7 +1,7 @@
 {
-  "a.out.js": 268033,
+  "a.out.js": 267889,
   "a.out.nodebug.wasm": 587151,
-  "total": 855184,
+  "total": 855040,
   "sent": [
     "IMG_Init",
     "IMG_Load",