diff --git a/ChangeLog.md b/ChangeLog.md index fd7f53ca469a9..5728cfd5b67db 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -20,6 +20,9 @@ See docs/process.md for more on how version tagging works. 5.0.3 (in development) ---------------------- +- The low level FS.write API now only accepts TypedArray. The higher level + writeFile and createDataFile file still also accept string and Array. + (#26413) - Warn on usage of the deprecated `EMSCRIPTEN` macro (`__EMSCRIPTEN__` should be used instead). (#26381) - The `-sRELOCATABLE` setting was effectively removed (moved to legacy diff --git a/src/lib/libcore.js b/src/lib/libcore.js index 29c4794c6a39e..fa3c9654d77e8 100644 --- a/src/lib/libcore.js +++ b/src/lib/libcore.js @@ -2193,7 +2193,7 @@ addToLibrary({ #endif FS.createPath('/', PATH.dirname(name), true, true); // canOwn this data in the filesystem, it is a slice of wasm memory that will never change - FS.createDataFile(name, null, HEAP8.subarray(content, content + len), true, true, true); + FS.createDataFile(name, null, HEAP8.subarray(content, content + len), true, true, /*canOwn=*/true); } while ({{{ makeGetValue('ptr', '0', '*') }}}); #if RUNTIME_DEBUG dbg('done preloading data files'); diff --git a/src/lib/libfs.js b/src/lib/libfs.js index 196866d2c31ab..d148c84f5a2d2 100644 --- a/src/lib/libfs.js +++ b/src/lib/libfs.js @@ -7,6 +7,7 @@ var LibraryFS = { $FS__deps: ['$randomFill', '$PATH', '$PATH_FS', '$TTY', '$MEMFS', '$FS_modeStringToFlags', + '$FS_fileDataToTypedArray', '$FS_getMode', '$intArrayFromString', #if LibraryManager.has('libidbfs.js') @@ -1253,9 +1254,13 @@ FS.staticInit();`; #endif return bytesRead; }, + /** + * @param {TypedArray} buffer + */ write(stream, buffer, offset, length, position, canOwn) { #if ASSERTIONS assert(offset >= 0); + assert(buffer.subarray, 'FS.write expects a TypedArray'); #endif if (length < 0 || position < 0) { throw new FS.ErrnoError({{{ cDefs.EINVAL }}}); @@ -1346,17 +1351,14 @@ FS.staticInit();`; FS.close(stream); return buf; }, + /** + * @param {TypedArray|Array|string} data + */ writeFile(path, data, opts = {}) { opts.flags = opts.flags || {{{ cDefs.O_TRUNC | cDefs.O_CREAT | cDefs.O_WRONLY }}}; var stream = FS.open(path, opts.flags, opts.mode); - if (typeof data == 'string') { - data = new Uint8Array(intArrayFromString(data, true)); - } - if (ArrayBuffer.isView(data)) { - FS.write(stream, data, 0, data.byteLength, undefined, opts.canOwn); - } else { - abort('Unsupported data type'); - } + data = FS_fileDataToTypedArray(data); + FS.write(stream, data, 0, data.byteLength, undefined, opts.canOwn); FS.close(stream); }, @@ -1604,6 +1606,9 @@ FS.staticInit();`; var mode = FS_getMode(canRead, canWrite); return FS.create(path, mode); }, + /** + * @param {TypedArray|Array|string=} data + */ createDataFile(parent, name, data, canRead, canWrite, canOwn) { var path = name; if (parent) { @@ -1613,11 +1618,7 @@ FS.staticInit();`; var mode = FS_getMode(canRead, canWrite); var node = FS.create(path, mode); if (data) { - if (typeof data == 'string') { - var arr = new Array(data.length); - for (var i = 0, len = data.length; i < len; ++i) arr[i] = data.charCodeAt(i); - data = arr; - } + data = FS_fileDataToTypedArray(data); // make sure we can write to the file FS.chmod(node, mode | {{{ cDefs.S_IWUGO }}}); var stream = FS.open(node, {{{ cDefs.O_TRUNC | cDefs.O_CREAT | cDefs.O_WRONLY }}}); diff --git a/src/lib/libfs_shared.js b/src/lib/libfs_shared.js index a80f7332157ad..838b52309701a 100644 --- a/src/lib/libfs_shared.js +++ b/src/lib/libfs_shared.js @@ -107,6 +107,16 @@ addToLibrary({ return mode; }, + $FS_fileDataToTypedArray: (data) => { + if (typeof data == 'string') { + data = intArrayFromString(data, true); + } + if (!data.subarray) { + data = new Uint8Array(data); + } + return data; + }, + $FS_stdin_getChar_buffer: [], // getChar has 3 particular return values: diff --git a/src/lib/libmemfs.js b/src/lib/libmemfs.js index cb9c60c6028aa..cfa5b56be7e00 100644 --- a/src/lib/libmemfs.js +++ b/src/lib/libmemfs.js @@ -248,16 +248,23 @@ addToLibrary({ return size; }, - // Writes the byte range (buffer[offset], buffer[offset+length]) to offset 'position' into the file pointed by 'stream' - // canOwn: A boolean that tells if this function can take ownership of the passed in buffer from the subbuffer portion - // that the typed array view 'buffer' points to. The underlying ArrayBuffer can be larger than that, but - // canOwn=true will not take ownership of the portion outside the bytes addressed by the view. This means that - // with canOwn=true, creating a copy of the bytes is avoided, but the caller shouldn't touch the passed in range - // of bytes anymore since their contents now represent file data inside the filesystem. + /** + * Writes the byte range (buffer[offset], buffer[offset+length]) to offset + * 'position' into the file pointed by 'stream'. + * @param {TypedArray} buffer + * @param {boolean=} canOwn - A boolean that tells if this function can + * take ownership of the passed in buffer from the subbuffer portion + * that the typed array view 'buffer' points to. The underlying + * ArrayBuffer can be larger than that, but canOwn=true will not take + * ownership of the portion outside the bytes addressed by the view. + * This means that with canOwn=true, creating a copy of the bytes is + * avoided, but the caller shouldn't touch the passed in range of + * bytes anymore since their contents now represent file data inside + * the filesystem. + */ write(stream, buffer, offset, length, position, canOwn) { #if ASSERTIONS - // The data buffer should be a typed array view - assert(!(buffer instanceof ArrayBuffer)); + assert(buffer.subarray, 'FS.write expects a TypedArray'); #endif #if ALLOW_MEMORY_GROWTH // If the buffer is located in main memory (HEAP), and if @@ -273,35 +280,21 @@ addToLibrary({ var node = stream.node; node.mtime = node.ctime = Date.now(); - if (buffer.subarray) { // This write is from a typed array to a typed array? - if (canOwn) { + if (canOwn) { #if ASSERTIONS - assert(position === 0, 'canOwn must imply no weird position inside the file'); + assert(position === 0, 'canOwn must imply no weird position inside the file'); #endif - node.contents = buffer.subarray(offset, offset + length); - node.usedBytes = length; - return length; - } else if (node.usedBytes === 0 && position === 0) { // If this is a simple first write to an empty file, do a fast set since we don't need to care about old data. - node.contents = buffer.slice(offset, offset + length); - node.usedBytes = length; - return length; - } else if (position + length <= node.usedBytes) { // Writing to an already allocated and used subrange of the file? - node.contents.set(buffer.subarray(offset, offset + length), position); - return length; - } - } - - // Appending to an existing file and we need to reallocate, or source data did not come as a typed array. - MEMFS.expandFileStorage(node, position+length); - if (buffer.subarray) { + node.contents = buffer.subarray(offset, offset + length); + node.usedBytes = length; + } else if (node.usedBytes === 0 && position === 0) { // If this is a simple first write to an empty file, do a fast set since we don't need to care about old data. + node.contents = buffer.slice(offset, offset + length); + node.usedBytes = length; + } else { + MEMFS.expandFileStorage(node, position+length); // Use typed array write which is available. node.contents.set(buffer.subarray(offset, offset + length), position); - } else { - for (var i = 0; i < length; i++) { - node.contents[position + i] = buffer[offset + i]; // Or fall back to manual write if not. - } + node.usedBytes = Math.max(node.usedBytes, position + length); } - node.usedBytes = Math.max(node.usedBytes, position + length); return length; }, diff --git a/src/lib/libwasmfs.js b/src/lib/libwasmfs.js index d168e9f6acde7..5a4f29ede10c3 100644 --- a/src/lib/libwasmfs.js +++ b/src/lib/libwasmfs.js @@ -509,20 +509,17 @@ addToLibrary({ return FS_mknod(path, mode, 0); }, - $FS_writeFile__deps: ['_wasmfs_write_file', '$stackSave', '$stackRestore', 'malloc', 'free'], + $FS_writeFile__deps: ['$FS_fileDataToTypedArray', '_wasmfs_write_file', '$stackSave', '$stackRestore', 'malloc', 'free'], $FS_writeFile: (path, data) => { var sp = stackSave(); var pathBuffer = stringToUTF8OnStack(path); - var len = typeof data == 'string' ? lengthBytesUTF8(data) + 1 : data.length; + data = FS_fileDataToTypedArray(data); + var len = data.length; var dataBuffer = _malloc(len); #if ASSERTIONS assert(dataBuffer); #endif - if (typeof data == 'string') { - len = stringToUTF8(data, dataBuffer, len); - } else { - HEAPU8.set(data, dataBuffer); - } + HEAPU8.set(data, dataBuffer); var ret = __wasmfs_write_file(pathBuffer, dataBuffer, len); _free(dataBuffer); stackRestore(sp); diff --git a/test/codesize/test_codesize_cxx_ctors1.json b/test/codesize/test_codesize_cxx_ctors1.json index b81c460b78f7e..54988716516f0 100644 --- a/test/codesize/test_codesize_cxx_ctors1.json +++ b/test/codesize/test_codesize_cxx_ctors1.json @@ -1,10 +1,10 @@ { - "a.out.js": 19334, - "a.out.js.gz": 8012, + "a.out.js": 19214, + "a.out.js.gz": 7982, "a.out.nodebug.wasm": 132865, "a.out.nodebug.wasm.gz": 49889, - "total": 152199, - "total_gz": 57901, + "total": 152079, + "total_gz": 57871, "sent": [ "__cxa_throw", "_abort_js", diff --git a/test/codesize/test_codesize_cxx_ctors2.json b/test/codesize/test_codesize_cxx_ctors2.json index 45575af3fbc93..4de0debeda7dd 100644 --- a/test/codesize/test_codesize_cxx_ctors2.json +++ b/test/codesize/test_codesize_cxx_ctors2.json @@ -1,10 +1,10 @@ { - "a.out.js": 19311, - "a.out.js.gz": 7998, + "a.out.js": 19191, + "a.out.js.gz": 7968, "a.out.nodebug.wasm": 132285, "a.out.nodebug.wasm.gz": 49542, - "total": 151596, - "total_gz": 57540, + "total": 151476, + "total_gz": 57510, "sent": [ "__cxa_throw", "_abort_js", diff --git a/test/codesize/test_codesize_cxx_except.json b/test/codesize/test_codesize_cxx_except.json index 4a858c94f0f87..b3169d1dc9f6e 100644 --- a/test/codesize/test_codesize_cxx_except.json +++ b/test/codesize/test_codesize_cxx_except.json @@ -1,10 +1,10 @@ { - "a.out.js": 22995, - "a.out.js.gz": 8989, + "a.out.js": 22875, + "a.out.js.gz": 8959, "a.out.nodebug.wasm": 172743, "a.out.nodebug.wasm.gz": 57374, - "total": 195738, - "total_gz": 66363, + "total": 195618, + "total_gz": 66333, "sent": [ "__cxa_begin_catch", "__cxa_end_catch", diff --git a/test/codesize/test_codesize_cxx_except_wasm.json b/test/codesize/test_codesize_cxx_except_wasm.json index 4d817c28b7be8..ce9b1467a28b3 100644 --- a/test/codesize/test_codesize_cxx_except_wasm.json +++ b/test/codesize/test_codesize_cxx_except_wasm.json @@ -1,10 +1,10 @@ { - "a.out.js": 19145, - "a.out.js.gz": 7934, + "a.out.js": 19025, + "a.out.js.gz": 7906, "a.out.nodebug.wasm": 148138, "a.out.nodebug.wasm.gz": 55261, - "total": 167283, - "total_gz": 63195, + "total": 167163, + "total_gz": 63167, "sent": [ "_abort_js", "_tzset_js", diff --git a/test/codesize/test_codesize_cxx_except_wasm_legacy.json b/test/codesize/test_codesize_cxx_except_wasm_legacy.json index eae00c3da2fa4..52a6429d34191 100644 --- a/test/codesize/test_codesize_cxx_except_wasm_legacy.json +++ b/test/codesize/test_codesize_cxx_except_wasm_legacy.json @@ -1,10 +1,10 @@ { - "a.out.js": 19219, - "a.out.js.gz": 7954, + "a.out.js": 19099, + "a.out.js.gz": 7926, "a.out.nodebug.wasm": 145944, "a.out.nodebug.wasm.gz": 54887, - "total": 165163, - "total_gz": 62841, + "total": 165043, + "total_gz": 62813, "sent": [ "_abort_js", "_tzset_js", diff --git a/test/codesize/test_codesize_cxx_lto.json b/test/codesize/test_codesize_cxx_lto.json index 9ed7da2c85026..b171846c1991f 100644 --- a/test/codesize/test_codesize_cxx_lto.json +++ b/test/codesize/test_codesize_cxx_lto.json @@ -1,10 +1,10 @@ { - "a.out.js": 18682, - "a.out.js.gz": 7697, + "a.out.js": 18562, + "a.out.js.gz": 7670, "a.out.nodebug.wasm": 102126, "a.out.nodebug.wasm.gz": 39426, - "total": 120808, - "total_gz": 47123, + "total": 120688, + "total_gz": 47096, "sent": [ "a (emscripten_resize_heap)", "b (_setitimer_js)", diff --git a/test/codesize/test_codesize_cxx_mangle.json b/test/codesize/test_codesize_cxx_mangle.json index 7861934ee3137..5a79291878812 100644 --- a/test/codesize/test_codesize_cxx_mangle.json +++ b/test/codesize/test_codesize_cxx_mangle.json @@ -1,10 +1,10 @@ { - "a.out.js": 23045, - "a.out.js.gz": 9008, + "a.out.js": 22925, + "a.out.js.gz": 8978, "a.out.nodebug.wasm": 239177, "a.out.nodebug.wasm.gz": 79774, - "total": 262222, - "total_gz": 88782, + "total": 262102, + "total_gz": 88752, "sent": [ "__cxa_begin_catch", "__cxa_end_catch", diff --git a/test/codesize/test_codesize_cxx_noexcept.json b/test/codesize/test_codesize_cxx_noexcept.json index a55086a4d111d..26057d45ca943 100644 --- a/test/codesize/test_codesize_cxx_noexcept.json +++ b/test/codesize/test_codesize_cxx_noexcept.json @@ -1,10 +1,10 @@ { - "a.out.js": 19334, - "a.out.js.gz": 8012, + "a.out.js": 19214, + "a.out.js.gz": 7982, "a.out.nodebug.wasm": 134872, "a.out.nodebug.wasm.gz": 50724, - "total": 154206, - "total_gz": 58736, + "total": 154086, + "total_gz": 58706, "sent": [ "__cxa_throw", "_abort_js", diff --git a/test/codesize/test_codesize_file_preload.expected.js b/test/codesize/test_codesize_file_preload.expected.js index 91e332a2e6b7f..a38f00ebe5427 100644 --- a/test/codesize/test_codesize_file_preload.expected.js +++ b/test/codesize/test_codesize_file_preload.expected.js @@ -1196,34 +1196,19 @@ var MEMFS = { if (!length) return 0; var node = stream.node; node.mtime = node.ctime = Date.now(); - if (buffer.subarray) { - // This write is from a typed array to a typed array? - if (canOwn) { - node.contents = buffer.subarray(offset, offset + length); - node.usedBytes = length; - return length; - } else if (node.usedBytes === 0 && position === 0) { - // If this is a simple first write to an empty file, do a fast set since we don't need to care about old data. - node.contents = buffer.slice(offset, offset + length); - node.usedBytes = length; - return length; - } else if (position + length <= node.usedBytes) { - // Writing to an already allocated and used subrange of the file? - node.contents.set(buffer.subarray(offset, offset + length), position); - return length; - } - } - // Appending to an existing file and we need to reallocate, or source data did not come as a typed array. - MEMFS.expandFileStorage(node, position + length); - if (buffer.subarray) { + if (canOwn) { + node.contents = buffer.subarray(offset, offset + length); + node.usedBytes = length; + } else if (node.usedBytes === 0 && position === 0) { + // If this is a simple first write to an empty file, do a fast set since we don't need to care about old data. + node.contents = buffer.slice(offset, offset + length); + node.usedBytes = length; + } else { + MEMFS.expandFileStorage(node, position + length); // Use typed array write which is available. node.contents.set(buffer.subarray(offset, offset + length), position); - } else { - for (var i = 0; i < length; i++) { - node.contents[position + i] = buffer[offset + i]; - } + node.usedBytes = Math.max(node.usedBytes, position + length); } - node.usedBytes = Math.max(node.usedBytes, position + length); return length; }, llseek(stream, offset, whence) { @@ -1301,6 +1286,16 @@ var FS_modeStringToFlags = str => { return flags; }; +var FS_fileDataToTypedArray = data => { + if (typeof data == "string") { + data = intArrayFromString(data, true); + } + if (!data.subarray) { + data = new Uint8Array(data); + } + return data; +}; + var FS_getMode = (canRead, canWrite) => { var mode = 0; if (canRead) mode |= 292 | 73; @@ -2490,14 +2485,8 @@ var FS = { writeFile(path, data, opts = {}) { opts.flags = opts.flags || 577; var stream = FS.open(path, opts.flags, opts.mode); - if (typeof data == "string") { - data = new Uint8Array(intArrayFromString(data, true)); - } - if (ArrayBuffer.isView(data)) { - FS.write(stream, data, 0, data.byteLength, undefined, opts.canOwn); - } else { - abort("Unsupported data type"); - } + data = FS_fileDataToTypedArray(data); + FS.write(stream, data, 0, data.byteLength, undefined, opts.canOwn); FS.close(stream); }, cwd: () => FS.currentPath, @@ -2724,11 +2713,7 @@ var FS = { var mode = FS_getMode(canRead, canWrite); var node = FS.create(path, mode); if (data) { - if (typeof data == "string") { - var arr = new Array(data.length); - for (var i = 0, len = data.length; i < len; ++i) arr[i] = data.charCodeAt(i); - data = arr; - } + data = FS_fileDataToTypedArray(data); // make sure we can write to the file FS.chmod(node, mode | 146); var stream = FS.open(node, 577); diff --git a/test/codesize/test_codesize_file_preload.json b/test/codesize/test_codesize_file_preload.json index 9579e98d43373..df9ad46824a2a 100644 --- a/test/codesize/test_codesize_file_preload.json +++ b/test/codesize/test_codesize_file_preload.json @@ -1,10 +1,10 @@ { - "a.out.js": 22289, - "a.out.js.gz": 9239, + "a.out.js": 22140, + "a.out.js.gz": 9187, "a.out.nodebug.wasm": 1681, "a.out.nodebug.wasm.gz": 960, - "total": 23970, - "total_gz": 10199, + "total": 23821, + "total_gz": 10147, "sent": [ "a (fd_write)" ], diff --git a/test/codesize/test_codesize_files_js_fs.json b/test/codesize/test_codesize_files_js_fs.json index 7a186e0ba07d9..ed6d9bccc4b10 100644 --- a/test/codesize/test_codesize_files_js_fs.json +++ b/test/codesize/test_codesize_files_js_fs.json @@ -1,10 +1,10 @@ { - "a.out.js": 17953, - "a.out.js.gz": 7339, + "a.out.js": 17833, + "a.out.js.gz": 7310, "a.out.nodebug.wasm": 381, "a.out.nodebug.wasm.gz": 260, - "total": 18334, - "total_gz": 7599, + "total": 18214, + "total_gz": 7570, "sent": [ "a (fd_write)", "b (fd_read)", diff --git a/test/codesize/test_codesize_hello_O0.json b/test/codesize/test_codesize_hello_O0.json index bf3b397ca4ba0..3ecd463f27324 100644 --- a/test/codesize/test_codesize_hello_O0.json +++ b/test/codesize/test_codesize_hello_O0.json @@ -1,10 +1,10 @@ { - "a.out.js": 24239, - "a.out.js.gz": 8714, + "a.out.js": 24263, + "a.out.js.gz": 8725, "a.out.nodebug.wasm": 15138, "a.out.nodebug.wasm.gz": 7455, - "total": 39377, - "total_gz": 16169, + "total": 39401, + "total_gz": 16180, "sent": [ "fd_write" ], diff --git a/test/codesize/test_codesize_hello_dylink.json b/test/codesize/test_codesize_hello_dylink.json index 7f24a526d7a58..2c172700f2583 100644 --- a/test/codesize/test_codesize_hello_dylink.json +++ b/test/codesize/test_codesize_hello_dylink.json @@ -1,10 +1,10 @@ { - "a.out.js": 26376, - "a.out.js.gz": 11261, + "a.out.js": 26256, + "a.out.js.gz": 11231, "a.out.nodebug.wasm": 17730, "a.out.nodebug.wasm.gz": 8957, - "total": 44106, - "total_gz": 20218, + "total": 43986, + "total_gz": 20188, "sent": [ "__syscall_stat64", "emscripten_resize_heap", diff --git a/test/codesize/test_codesize_hello_dylink_all.json b/test/codesize/test_codesize_hello_dylink_all.json index 8dedb57b8f3dd..1a6cba8395cca 100644 --- a/test/codesize/test_codesize_hello_dylink_all.json +++ b/test/codesize/test_codesize_hello_dylink_all.json @@ -1,7 +1,7 @@ { - "a.out.js": 244462, + "a.out.js": 244330, "a.out.nodebug.wasm": 577692, - "total": 822154, + "total": 822022, "sent": [ "IMG_Init", "IMG_Load", diff --git a/test/codesize/test_codesize_minimal_O0.expected.js b/test/codesize/test_codesize_minimal_O0.expected.js index 08922367380bb..cdf31f4dc0ee5 100644 --- a/test/codesize/test_codesize_minimal_O0.expected.js +++ b/test/codesize/test_codesize_minimal_O0.expected.js @@ -1051,6 +1051,7 @@ Module['FS_createPreloadedFile'] = FS.createPreloadedFile; 'FS_preloadFile', 'FS_modeStringToFlags', 'FS_getMode', + 'FS_fileDataToTypedArray', 'FS_stdin_getChar', 'FS_mkdirTree', '_setNetworkCallback', diff --git a/test/codesize/test_codesize_minimal_O0.json b/test/codesize/test_codesize_minimal_O0.json index 8baef8820d3cb..ca88e74a19508 100644 --- a/test/codesize/test_codesize_minimal_O0.json +++ b/test/codesize/test_codesize_minimal_O0.json @@ -1,10 +1,10 @@ { - "a.out.js": 19425, - "a.out.js.gz": 6992, + "a.out.js": 19449, + "a.out.js.gz": 7006, "a.out.nodebug.wasm": 1136, "a.out.nodebug.wasm.gz": 656, - "total": 20561, - "total_gz": 7648, + "total": 20585, + "total_gz": 7662, "sent": [], "imports": [], "exports": [ diff --git a/test/codesize/test_unoptimized_code_size.json b/test/codesize/test_unoptimized_code_size.json index 197c9407eb0b3..635a347810bec 100644 --- a/test/codesize/test_unoptimized_code_size.json +++ b/test/codesize/test_unoptimized_code_size.json @@ -1,16 +1,16 @@ { - "hello_world.js": 56978, - "hello_world.js.gz": 17726, + "hello_world.js": 57007, + "hello_world.js.gz": 17736, "hello_world.wasm": 15138, "hello_world.wasm.gz": 7455, "no_asserts.js": 26576, "no_asserts.js.gz": 8881, "no_asserts.wasm": 12187, "no_asserts.wasm.gz": 5984, - "strict.js": 54796, - "strict.js.gz": 17024, + "strict.js": 54825, + "strict.js.gz": 17039, "strict.wasm": 15138, "strict.wasm.gz": 7450, - "total": 180813, - "total_gz": 64520 + "total": 180871, + "total_gz": 64545 }