diff --git a/Cargo.lock b/Cargo.lock index e890d367f22c..f4eabc03a260 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,7 +287,7 @@ dependencies = [ [[package]] name = "byte-array-literals" -version = "48.0.0" +version = "48.0.1" [[package]] name = "byteorder" @@ -348,9 +348,9 @@ dependencies = [ [[package]] name = "cap-fs-ext" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78e5a3368ae89b7cb68186411452b4b9fac8b41be9c19bf3f47c2d2c8e36e6b" +checksum = "56ff379b70af8e08307a8f65e7040c7301cb4a572538ade16b4984f0da77847f" dependencies = [ "cap-primitives", "cap-std", @@ -360,9 +360,9 @@ dependencies = [ [[package]] name = "cap-primitives" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdadbd7c002d3a484b35243669abdae85a0ebaded5a61117169dc3400f9a7ff0" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" dependencies = [ "ambient-authority", "fs-set-times", @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "cap-std" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7281235d6e96d3544ca18bba9049be92f4190f8d923e3caef1b5f66cfa752608" +checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" dependencies = [ "cap-primitives", "io-extras", @@ -673,7 +673,7 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-codegen", "cranelift-frontend", @@ -686,7 +686,7 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.135.0" +version = "0.135.1" dependencies = [ "arbitrary", "arbtest", @@ -705,14 +705,14 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64-meta" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -720,7 +720,7 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.135.0" +version = "0.135.1" dependencies = [ "arbitrary", "serde", @@ -730,7 +730,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.135.0" +version = "0.135.1" dependencies = [ "anyhow", "bumpalo", @@ -767,7 +767,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -778,18 +778,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.135.0" +version = "0.135.1" [[package]] name = "cranelift-control" -version = "0.135.0" +version = "0.135.1" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-bitset", "serde", @@ -831,7 +831,7 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-codegen", "env_logger 0.11.5", @@ -856,7 +856,7 @@ dependencies = [ [[package]] name = "cranelift-interpreter" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -870,7 +870,7 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.135.0" +version = "0.135.1" dependencies = [ "codespan-reporting", "log", @@ -954,7 +954,7 @@ dependencies = [ [[package]] name = "cranelift-jit" -version = "0.135.0" +version = "0.135.1" dependencies = [ "anyhow", "cranelift", @@ -976,7 +976,7 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.135.0" +version = "0.135.1" dependencies = [ "anyhow", "cranelift-codegen", @@ -988,7 +988,7 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.135.0" +version = "0.135.1" dependencies = [ "cranelift-codegen", "libc", @@ -997,7 +997,7 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.135.0" +version = "0.135.1" dependencies = [ "anyhow", "cranelift-codegen", @@ -1013,7 +1013,7 @@ dependencies = [ [[package]] name = "cranelift-reader" -version = "0.135.0" +version = "0.135.1" dependencies = [ "anyhow", "cranelift-codegen", @@ -1023,7 +1023,7 @@ dependencies = [ [[package]] name = "cranelift-serde" -version = "0.135.0" +version = "0.135.1" dependencies = [ "clap", "cranelift-codegen", @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.135.0" +version = "0.135.1" [[package]] name = "cranelift-tools" @@ -1294,7 +1294,7 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "embedding" -version = "48.0.0" +version = "48.0.1" dependencies = [ "dlmalloc", "raw-cpuid", @@ -2462,7 +2462,7 @@ dependencies = [ [[package]] name = "min-platform-host" -version = "48.0.0" +version = "48.0.1" dependencies = [ "libloading", "object 0.39.0", @@ -2987,7 +2987,7 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "arbitrary", @@ -3011,7 +3011,7 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "48.0.0" +version = "48.0.1" dependencies = [ "proc-macro2", "quote", @@ -4274,7 +4274,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "verify-component-adapter" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "wasmparser 0.254.0", @@ -4348,7 +4348,7 @@ dependencies = [ [[package]] name = "wasi-preview1-component-adapter" -version = "48.0.0" +version = "48.0.1" dependencies = [ "bitflags 2.11.1", "byte-array-literals", @@ -4580,7 +4580,7 @@ dependencies = [ [[package]] name = "wasmtime" -version = "48.0.0" +version = "48.0.1" dependencies = [ "addr2line 0.26.0", "async-trait", @@ -4641,7 +4641,7 @@ dependencies = [ [[package]] name = "wasmtime-bench-api" -version = "48.0.0" +version = "48.0.1" dependencies = [ "clap", "shuffling-allocator", @@ -4655,14 +4655,14 @@ dependencies = [ [[package]] name = "wasmtime-c-api" -version = "48.0.0" +version = "48.0.1" dependencies = [ "wasmtime-c-api-impl", ] [[package]] name = "wasmtime-c-api-impl" -version = "48.0.0" +version = "48.0.1" dependencies = [ "async-trait", "bytes", @@ -4681,7 +4681,7 @@ dependencies = [ [[package]] name = "wasmtime-cli" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "async-trait", @@ -4758,7 +4758,7 @@ dependencies = [ [[package]] name = "wasmtime-cli-flags" -version = "48.0.0" +version = "48.0.1" dependencies = [ "clap", "file-per-thread-logger", @@ -4772,7 +4772,7 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "arbitrary", @@ -4889,7 +4889,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-c-api-macros" -version = "48.0.0" +version = "48.0.1" dependencies = [ "proc-macro2", "quote", @@ -4897,7 +4897,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-cache" -version = "48.0.0" +version = "48.0.1" dependencies = [ "base64", "directories-next", @@ -4918,7 +4918,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "component-macro-test-helpers", @@ -4938,11 +4938,11 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "48.0.0" +version = "48.0.1" [[package]] name = "wasmtime-internal-core" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "hashbrown 0.17.0", @@ -4952,7 +4952,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "48.0.0" +version = "48.0.1" dependencies = [ "cranelift-codegen", "cranelift-control", @@ -4976,7 +4976,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-debugger" -version = "48.0.0" +version = "48.0.1" dependencies = [ "async-trait", "env_logger 0.11.5", @@ -4989,7 +4989,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-explorer" -version = "48.0.0" +version = "48.0.1" dependencies = [ "capstone", "serde", @@ -5003,7 +5003,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "48.0.0" +version = "48.0.1" dependencies = [ "backtrace", "cc", @@ -5016,7 +5016,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-gdbstub-component" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "clap", @@ -5032,11 +5032,11 @@ dependencies = [ [[package]] name = "wasmtime-internal-gdbstub-component-artifact" -version = "48.0.0" +version = "48.0.1" [[package]] name = "wasmtime-internal-jit-debug" -version = "48.0.0" +version = "48.0.1" dependencies = [ "cc", "object 0.39.0", @@ -5046,7 +5046,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "48.0.0" +version = "48.0.1" dependencies = [ "libc", "wasmtime-internal-core", @@ -5055,7 +5055,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "48.0.0" +version = "48.0.1" dependencies = [ "cranelift-codegen", "log", @@ -5065,7 +5065,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "48.0.0" +version = "48.0.1" dependencies = [ "proc-macro2", "quote", @@ -5074,7 +5074,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "48.0.0" +version = "48.0.1" dependencies = [ "cranelift-codegen", "gimli 0.33.0", @@ -5089,7 +5089,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "48.0.0" +version = "48.0.1" dependencies = [ "anyhow", "bitflags 2.11.1", @@ -5101,7 +5101,7 @@ dependencies = [ [[package]] name = "wasmtime-internal-wmemcheck" -version = "48.0.0" +version = "48.0.1" [[package]] name = "wasmtime-test-macros" @@ -5116,7 +5116,7 @@ dependencies = [ [[package]] name = "wasmtime-test-util" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arbitrary", "arbtest", @@ -5139,7 +5139,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "48.0.0" +version = "48.0.1" dependencies = [ "async-trait", "bitflags 2.11.1", @@ -5167,7 +5167,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi-config" -version = "48.0.0" +version = "48.0.1" dependencies = [ "test-programs-artifacts", "tokio", @@ -5177,7 +5177,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi-http" -version = "48.0.0" +version = "48.0.1" dependencies = [ "async-trait", "base64", @@ -5210,7 +5210,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "48.0.0" +version = "48.0.1" dependencies = [ "async-trait", "bytes", @@ -5221,7 +5221,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi-keyvalue" -version = "48.0.0" +version = "48.0.1" dependencies = [ "test-programs-artifacts", "tokio", @@ -5231,7 +5231,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi-nn" -version = "48.0.0" +version = "48.0.1" dependencies = [ "libtest-mimic", "openvino", @@ -5250,7 +5250,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi-tls" -version = "48.0.0" +version = "48.0.1" dependencies = [ "bytes", "futures", @@ -5270,7 +5270,7 @@ dependencies = [ [[package]] name = "wasmtime-wast" -version = "48.0.0" +version = "48.0.1" dependencies = [ "json-from-wast", "log", @@ -5284,7 +5284,7 @@ dependencies = [ [[package]] name = "wasmtime-wizer" -version = "48.0.0" +version = "48.0.1" dependencies = [ "clap", "criterion", @@ -5365,7 +5365,7 @@ dependencies = [ [[package]] name = "wiggle" -version = "48.0.0" +version = "48.0.1" dependencies = [ "bitflags 2.11.1", "proptest", @@ -5381,7 +5381,7 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "48.0.0" +version = "48.0.1" dependencies = [ "heck", "proc-macro2", @@ -5393,7 +5393,7 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "48.0.0" +version = "48.0.1" dependencies = [ "proc-macro2", "quote", @@ -5448,7 +5448,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "48.0.0" +version = "48.0.1" dependencies = [ "cranelift-assembler-x64", "cranelift-codegen", diff --git a/Cargo.toml b/Cargo.toml index bdfba85d5c2a..83b09c974232 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -219,7 +219,7 @@ exclude = [ ] [workspace.package] -version = "48.0.0" +version = "48.0.1" authors = ["The Wasmtime Project Developers"] edition = "2024" # Wasmtime's current policy is that this number can be no larger than the @@ -280,17 +280,17 @@ extra_unused_type_parameters = 'warn' # tooling but aren't intended to be widely depended on. # # All of these crates are supported though in the sense that -wasmtime = { path = "crates/wasmtime", version = "48.0.0", default-features = false } -wasmtime-cli-flags = { path = "crates/cli-flags", version = "=48.0.0" } -wasmtime-environ = { path = "crates/environ", version = "=48.0.0" } -wasmtime-wasi = { path = "crates/wasi", version = "48.0.0", default-features = false } -wasmtime-wasi-io = { path = "crates/wasi-io", version = "48.0.0", default-features = false } -wasmtime-wasi-http = { path = "crates/wasi-http", version = "48.0.0", default-features = false } -wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "48.0.0" } -wasmtime-wasi-config = { path = "crates/wasi-config", version = "48.0.0" } -wasmtime-wasi-keyvalue = { path = "crates/wasi-keyvalue", version = "48.0.0" } -wasmtime-wasi-tls = { path = "crates/wasi-tls", version = "48.0.0" } -wasmtime-wast = { path = "crates/wast", version = "=48.0.0" } +wasmtime = { path = "crates/wasmtime", version = "48.0.1", default-features = false } +wasmtime-cli-flags = { path = "crates/cli-flags", version = "=48.0.1" } +wasmtime-environ = { path = "crates/environ", version = "=48.0.1" } +wasmtime-wasi = { path = "crates/wasi", version = "48.0.1", default-features = false } +wasmtime-wasi-io = { path = "crates/wasi-io", version = "48.0.1", default-features = false } +wasmtime-wasi-http = { path = "crates/wasi-http", version = "48.0.1", default-features = false } +wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "48.0.1" } +wasmtime-wasi-config = { path = "crates/wasi-config", version = "48.0.1" } +wasmtime-wasi-keyvalue = { path = "crates/wasi-keyvalue", version = "48.0.1" } +wasmtime-wasi-tls = { path = "crates/wasi-tls", version = "48.0.1" } +wasmtime-wast = { path = "crates/wast", version = "=48.0.1" } # Internal Wasmtime-specific crates. # @@ -299,54 +299,54 @@ wasmtime-wast = { path = "crates/wast", version = "=48.0.0" } # that these are internal unsupported crates for external use. These exist as # part of the project organization of other public crates in Wasmtime and are # otherwise not supported in terms of CVEs for example. -wasmtime-core = { path = "crates/core", version = "=48.0.0", package = 'wasmtime-internal-core' } -wasmtime-wmemcheck = { path = "crates/wmemcheck", version = "=48.0.0", package = 'wasmtime-internal-wmemcheck' } -wasmtime-c-api-macros = { path = "crates/c-api-macros", version = "=48.0.0", package = 'wasmtime-internal-c-api-macros' } -wasmtime-cache = { path = "crates/cache", version = "=48.0.0", package = 'wasmtime-internal-cache' } -wasmtime-cranelift = { path = "crates/cranelift", version = "=48.0.0", package = 'wasmtime-internal-cranelift' } -wasmtime-winch = { path = "crates/winch", version = "=48.0.0", package = 'wasmtime-internal-winch' } -wasmtime-explorer = { path = "crates/explorer", version = "=48.0.0", package = 'wasmtime-internal-explorer' } -wasmtime-fiber = { path = "crates/fiber", version = "=48.0.0", package = 'wasmtime-internal-fiber' } -wasmtime-jit-debug = { path = "crates/jit-debug", version = "=48.0.0", package = 'wasmtime-internal-jit-debug' } -wasmtime-component-util = { path = "crates/component-util", version = "=48.0.0", package = 'wasmtime-internal-component-util' } -wasmtime-component-macro = { path = "crates/component-macro", version = "=48.0.0", package = 'wasmtime-internal-component-macro' } -wasmtime-versioned-export-macros = { path = "crates/versioned-export-macros", version = "=48.0.0", package = 'wasmtime-internal-versioned-export-macros' } -wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=48.0.0", package = 'wasmtime-internal-jit-icache-coherence' } -wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=48.0.0", package = 'wasmtime-internal-wit-bindgen' } -wasmtime-unwinder = { path = "crates/unwinder", version = "=48.0.0", package = 'wasmtime-internal-unwinder' } -wasmtime-debugger = { path = "crates/debugger", version = "=48.0.0", package = "wasmtime-internal-debugger" } -gdbstub-component-artifact = { path = "crates/gdbstub-component/artifact", version = "=48.0.0", package = "wasmtime-internal-gdbstub-component-artifact" } -wasmtime-wizer = { path = "crates/wizer", version = "48.0.0" } +wasmtime-core = { path = "crates/core", version = "=48.0.1", package = 'wasmtime-internal-core' } +wasmtime-wmemcheck = { path = "crates/wmemcheck", version = "=48.0.1", package = 'wasmtime-internal-wmemcheck' } +wasmtime-c-api-macros = { path = "crates/c-api-macros", version = "=48.0.1", package = 'wasmtime-internal-c-api-macros' } +wasmtime-cache = { path = "crates/cache", version = "=48.0.1", package = 'wasmtime-internal-cache' } +wasmtime-cranelift = { path = "crates/cranelift", version = "=48.0.1", package = 'wasmtime-internal-cranelift' } +wasmtime-winch = { path = "crates/winch", version = "=48.0.1", package = 'wasmtime-internal-winch' } +wasmtime-explorer = { path = "crates/explorer", version = "=48.0.1", package = 'wasmtime-internal-explorer' } +wasmtime-fiber = { path = "crates/fiber", version = "=48.0.1", package = 'wasmtime-internal-fiber' } +wasmtime-jit-debug = { path = "crates/jit-debug", version = "=48.0.1", package = 'wasmtime-internal-jit-debug' } +wasmtime-component-util = { path = "crates/component-util", version = "=48.0.1", package = 'wasmtime-internal-component-util' } +wasmtime-component-macro = { path = "crates/component-macro", version = "=48.0.1", package = 'wasmtime-internal-component-macro' } +wasmtime-versioned-export-macros = { path = "crates/versioned-export-macros", version = "=48.0.1", package = 'wasmtime-internal-versioned-export-macros' } +wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=48.0.1", package = 'wasmtime-internal-jit-icache-coherence' } +wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=48.0.1", package = 'wasmtime-internal-wit-bindgen' } +wasmtime-unwinder = { path = "crates/unwinder", version = "=48.0.1", package = 'wasmtime-internal-unwinder' } +wasmtime-debugger = { path = "crates/debugger", version = "=48.0.1", package = "wasmtime-internal-debugger" } +gdbstub-component-artifact = { path = "crates/gdbstub-component/artifact", version = "=48.0.1", package = "wasmtime-internal-gdbstub-component-artifact" } +wasmtime-wizer = { path = "crates/wizer", version = "48.0.1" } # Miscellaneous crates without a `wasmtime-*` prefix in their name but still # used in the `wasmtime-*` family of crates depending on various features/etc. -wiggle = { path = "crates/wiggle", version = "=48.0.0", default-features = false } -wiggle-macro = { path = "crates/wiggle/macro", version = "=48.0.0" } -wiggle-generate = { path = "crates/wiggle/generate", version = "=48.0.0" } -pulley-interpreter = { path = 'pulley', version = "=48.0.0" } -pulley-macros = { path = 'pulley/macros', version = "=48.0.0" } +wiggle = { path = "crates/wiggle", version = "=48.0.1", default-features = false } +wiggle-macro = { path = "crates/wiggle/macro", version = "=48.0.1" } +wiggle-generate = { path = "crates/wiggle/generate", version = "=48.0.1" } +pulley-interpreter = { path = 'pulley', version = "=48.0.1" } +pulley-macros = { path = 'pulley/macros', version = "=48.0.1" } # Cranelift crates in this workspace -cranelift-assembler-x64 = { path = "cranelift/assembler-x64", version = "0.135.0" } -cranelift-codegen = { path = "cranelift/codegen", version = "0.135.0", default-features = false, features = ["unwind"] } -cranelift-frontend = { path = "cranelift/frontend", version = "0.135.0" } -cranelift-entity = { path = "cranelift/entity", version = "0.135.0" } -cranelift-native = { path = "cranelift/native", version = "0.135.0" } -cranelift-module = { path = "cranelift/module", version = "0.135.0" } -cranelift-interpreter = { path = "cranelift/interpreter", version = "0.135.0" } -cranelift-reader = { path = "cranelift/reader", version = "0.135.0" } +cranelift-assembler-x64 = { path = "cranelift/assembler-x64", version = "0.135.1" } +cranelift-codegen = { path = "cranelift/codegen", version = "0.135.1", default-features = false, features = ["unwind"] } +cranelift-frontend = { path = "cranelift/frontend", version = "0.135.1" } +cranelift-entity = { path = "cranelift/entity", version = "0.135.1" } +cranelift-native = { path = "cranelift/native", version = "0.135.1" } +cranelift-module = { path = "cranelift/module", version = "0.135.1" } +cranelift-interpreter = { path = "cranelift/interpreter", version = "0.135.1" } +cranelift-reader = { path = "cranelift/reader", version = "0.135.1" } cranelift-filetests = { path = "cranelift/filetests" } -cranelift-object = { path = "cranelift/object", version = "0.135.0" } -cranelift-jit = { path = "cranelift/jit", version = "0.135.0" } +cranelift-object = { path = "cranelift/object", version = "0.135.1" } +cranelift-jit = { path = "cranelift/jit", version = "0.135.1" } cranelift-fuzzgen = { path = "cranelift/fuzzgen" } -cranelift-bforest = { path = "cranelift/bforest", version = "0.135.0" } -cranelift-bitset = { path = "cranelift/bitset", version = "0.135.0" } -cranelift-control = { path = "cranelift/control", version = "0.135.0", default-features = false } -cranelift-srcgen = { path = "cranelift/srcgen", version = "0.135.0" } -cranelift = { path = "cranelift/umbrella", version = "0.135.0" } +cranelift-bforest = { path = "cranelift/bforest", version = "0.135.1" } +cranelift-bitset = { path = "cranelift/bitset", version = "0.135.1" } +cranelift-control = { path = "cranelift/control", version = "0.135.1", default-features = false } +cranelift-srcgen = { path = "cranelift/srcgen", version = "0.135.1" } +cranelift = { path = "cranelift/umbrella", version = "0.135.1" } # Winch crates in this workspace. -winch-codegen = { path = "winch/codegen", version = "=48.0.0" } +winch-codegen = { path = "winch/codegen", version = "=48.0.1" } # Internal crates not published to crates.io used in testing, builds, etc wasi-preview1-component-adapter = { path = "crates/wasi-preview1-component-adapter" } @@ -367,8 +367,8 @@ wasip1 = { version = "1.0.0", default-features = false } # Note that `cap-fs-ext` should be avoided where possible to use # `cap-primitives` instead. target-lexicon = "0.13.5" -cap-primitives = "4.0.2" -cap-fs-ext-avoid-using-this = { version = "4.0.2", package = 'cap-fs-ext' } +cap-primitives = "4.0.3" +cap-fs-ext-avoid-using-this = { version = "4.0.3", package = 'cap-fs-ext' } rustix = "1.1.4" # wit-bindgen: wit-bindgen = { version = "0.60.0", default-features = false } diff --git a/RELEASES.md b/RELEASES.md index 892016d80105..53e9f7c82a18 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,11 +1,153 @@ +## 48.0.1 + +Released 2026-08-24. + +### Fixed + +* Context slots in component compositions are now correctly managed. + [#14139](https://github.com/bytecodealliance/wasmtime/pull/14139) + +* The `Host` header is now set by default for HTTP requests sent with WASIp2. + [#14167](https://github.com/bytecodealliance/wasmtime/pull/14167) + +-------------------------------------------------------------------------------- + ## 48.0.0 -Unreleased. +Released 2026-08-20. ### Added +* Wasmtime has an initial implementation of the component model + fixed-length-lists feature. + [#12315](https://github.com/bytecodealliance/wasmtime/pull/12315) + +* Wasmtime's reflection of component imports/exports now exposes `external-id` + information. + [#13874](https://github.com/bytecodealliance/wasmtime/pull/13874) + +* Cranelift's alias analysis pass now eliminates dead stores. + [#13806](https://github.com/bytecodealliance/wasmtime/pull/13806) + [#13947](https://github.com/bytecodealliance/wasmtime/pull/13947) + +* Winch now supports the wasm SIMD proposal on AArch64. + [#13911](https://github.com/bytecodealliance/wasmtime/pull/13911) + [#13921](https://github.com/bytecodealliance/wasmtime/pull/13921) + [#13928](https://github.com/bytecodealliance/wasmtime/pull/13928) + [#13937](https://github.com/bytecodealliance/wasmtime/pull/13937) + [#13945](https://github.com/bytecodealliance/wasmtime/pull/13945) + [#13938](https://github.com/bytecodealliance/wasmtime/pull/13938) + [#13946](https://github.com/bytecodealliance/wasmtime/pull/13946) + ... + +* Cranelift now supports an AVX512-VNNI lowering for the dot-product wasm + instruction and `usdot` on AArch64. + [#14006](https://github.com/bytecodealliance/wasmtime/pull/14006) + [#14054](https://github.com/bytecodealliance/wasmtime/pull/14054) + +* Wasmtime's `bindgen!` macro now has an `include_component_type` option to + generate a `COMPONENT_TYPE` constant with the encoded type of the world being + bound. + [#14013](https://github.com/bytecodealliance/wasmtime/pull/14013) + +* Wasmtime now supports configurable fuel costs for variable-length wasm + opcodes. + [#13931](https://github.com/bytecodealliance/wasmtime/pull/13931) + ### Changed +* Host-implemented traits in `wasmtime-wasi-http` are now the same across + wasip2/wasip3 and no longer require separate implementations/structures. + [#13810](https://github.com/bytecodealliance/wasmtime/pull/13810) + [#13812](https://github.com/bytecodealliance/wasmtime/pull/13812) + [#13835](https://github.com/bytecodealliance/wasmtime/pull/13835) + +* Wasmtime will now use the `process_madvise` syscall on Linux where available + which can improve the performance of the pooling allocator when the + deallocation batch size is configured to larger than 1. + [#13830](https://github.com/bytecodealliance/wasmtime/pull/13830) + +* Synchronous cancellation of streams/futures/subtasks now traps if the waitable + was already in a waitable set. + [#13708](https://github.com/bytecodealliance/wasmtime/pull/13708) + +* Winch is now flagged to be compatible with component-model-async. + [#13845](https://github.com/bytecodealliance/wasmtime/pull/13845) + +* Wasmtime's pooling allocator now uses more sharding to reduce lock contention. + [#13840](https://github.com/bytecodealliance/wasmtime/pull/13840) + +* Wasmtime now requires Rust 1.95.0 to build. + [#13853](https://github.com/bytecodealliance/wasmtime/pull/13853) + +* Cranelift now has a uniform maximum size across all backends on the bounds of + a function's stack frame. + [#13783](https://github.com/bytecodealliance/wasmtime/pull/13783) + +* Wasmtime's `LinkerInstance` for components now supports being reopened to + gradually add more items. + [#13908](https://github.com/bytecodealliance/wasmtime/pull/13908) + +* The `wasmtime-wasi` crate's default configuration now denies creation of + TCP/UDP sockets by default. + [#13936](https://github.com/bytecodealliance/wasmtime/pull/13936) + +* Wasmtime now configures async task context fields on `realloc` calls to zero. + [#13949](https://github.com/bytecodealliance/wasmtime/pull/13949) + +* Work has continued on continuous verification of Cranelift's lowering rules + on AArch64. + [#13935](https://github.com/bytecodealliance/wasmtime/pull/13935) + [#13929](https://github.com/bytecodealliance/wasmtime/pull/13929) + [#13998](https://github.com/bytecodealliance/wasmtime/pull/13998) + +* Codegen for loads on AArch64 has been optimized slightly to improve sharing + common sub-expressions. + [#13766](https://github.com/bytecodealliance/wasmtime/pull/13766) + +* Work continues on implementing the stack-switching proposal. + [#11717](https://github.com/bytecodealliance/wasmtime/pull/11717) + [#13996](https://github.com/bytecodealliance/wasmtime/pull/13996) + [#14052](https://github.com/bytecodealliance/wasmtime/pull/14052) + +* Permissions for `wasi-filesystem` in the implementation of the `wasmtime-wasi` + crate have been simplified to either read-write or read-only for a directory. + [#14010](https://github.com/bytecodealliance/wasmtime/pull/14010) + +### Fixed + +* A late-drop of host-defined stream producers/consumers has been fixed. + [#13891](https://github.com/bytecodealliance/wasmtime/pull/13891) + +* Call hooks are now invoked around async yields when dealing with concurrent + execution. + [#13871](https://github.com/bytecodealliance/wasmtime/pull/13871) + +* Extreme filesystem timestamps no longer cause panics. + [#13894](https://github.com/bytecodealliance/wasmtime/pull/13894) + +* An erroneous trap was fixed where an async-delivered write-closed event was + sent to a future. + [#13914](https://github.com/bytecodealliance/wasmtime/pull/13914) + +* An erroneous trap where Wasmtime internally used the wrong thread id has been + fixed. + [#13926](https://github.com/bytecodealliance/wasmtime/pull/13926) + +* Cross-component instance streams are no longer erroneously flagged as being + intra-instance. + [#14018](https://github.com/bytecodealliance/wasmtime/pull/14018) + +* A panic in the `wasmtime` CLI with non-utf8 environment variables has been + fixed. + [#14017](https://github.com/bytecodealliance/wasmtime/pull/14017) + +* Atomic waits on big-endian hosts have been fixed. + [#14027](https://github.com/bytecodealliance/wasmtime/pull/14027) + +* Enabling MPK with CoW images has been fixed. + [#14076](https://github.com/bytecodealliance/wasmtime/pull/14076) + -------------------------------------------------------------------------------- Release notes for previous releases of Wasmtime can be found on the respective diff --git a/cranelift/assembler-x64/Cargo.toml b/cranelift/assembler-x64/Cargo.toml index b93f375f30ef..d18ea6bdebcd 100644 --- a/cranelift/assembler-x64/Cargo.toml +++ b/cranelift/assembler-x64/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "cranelift-assembler-x64" description = "A Cranelift-specific x64 assembler" -version = "0.135.0" +version = "0.135.1" license = "Apache-2.0 WITH LLVM-exception" edition.workspace = true rust-version.workspace = true @@ -25,7 +25,7 @@ arbtest = { workspace = true } capstone = { workspace = true } [build-dependencies] -cranelift-assembler-x64-meta = { path = "meta", version = "0.135.0" } +cranelift-assembler-x64-meta = { path = "meta", version = "0.135.1" } [lints] workspace = true diff --git a/cranelift/assembler-x64/meta/Cargo.toml b/cranelift/assembler-x64/meta/Cargo.toml index e309210e4939..d8a1c4ea64ea 100644 --- a/cranelift/assembler-x64/meta/Cargo.toml +++ b/cranelift/assembler-x64/meta/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "cranelift-assembler-x64-meta" description = "Generate a Cranelift-specific assembler for x64 instructions" -version = "0.135.0" +version = "0.135.1" license = "Apache-2.0 WITH LLVM-exception" edition.workspace = true rust-version.workspace = true diff --git a/cranelift/bforest/Cargo.toml b/cranelift/bforest/Cargo.toml index e19a68e3713c..4512c9c7abfc 100644 --- a/cranelift/bforest/Cargo.toml +++ b/cranelift/bforest/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-bforest" -version = "0.135.0" +version = "0.135.1" description = "A forest of B+-trees" license = "Apache-2.0 WITH LLVM-exception" documentation = "https://docs.rs/cranelift-bforest" diff --git a/cranelift/bitset/Cargo.toml b/cranelift/bitset/Cargo.toml index 80247097a056..6db1dfa4750a 100644 --- a/cranelift/bitset/Cargo.toml +++ b/cranelift/bitset/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-bitset" -version = "0.135.0" +version = "0.135.1" description = "Various bitset stuff for use inside Cranelift" license = "Apache-2.0 WITH LLVM-exception" documentation = "https://docs.rs/cranelift-bitset" diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 9235281ec986..fe0f963c0615 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-codegen" -version = "0.135.0" +version = "0.135.1" description = "Low-level code generator library" license = "Apache-2.0 WITH LLVM-exception" documentation = "https://docs.rs/cranelift-codegen" @@ -25,7 +25,7 @@ anyhow = { workspace = true, optional = true, features = ['std'] } bumpalo = { workspace = true } capstone = { workspace = true, optional = true } cranelift-assembler-x64 = { workspace = true } -cranelift-codegen-shared = { path = "./shared", version = "0.135.0" } +cranelift-codegen-shared = { path = "./shared", version = "0.135.1" } cranelift-entity = { workspace = true } cranelift-bforest = { workspace = true } cranelift-bitset = { workspace = true } @@ -58,8 +58,8 @@ proptest = { workspace = true } mutatis = { workspace = true } [build-dependencies] -cranelift-codegen-meta = { path = "meta", version = "0.135.0" } -cranelift-isle = { path = "../isle/isle", version = "=0.135.0" } +cranelift-codegen-meta = { path = "meta", version = "0.135.1" } +cranelift-isle = { path = "../isle/isle", version = "=0.135.1" } [features] default = ["std", "unwind", "host-arch", "timing"] diff --git a/cranelift/codegen/meta/Cargo.toml b/cranelift/codegen/meta/Cargo.toml index ed83810c5e42..5278a1b07fb2 100644 --- a/cranelift/codegen/meta/Cargo.toml +++ b/cranelift/codegen/meta/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "cranelift-codegen-meta" authors = ["The Cranelift Project Developers"] -version = "0.135.0" +version = "0.135.1" description = "Metaprogram for cranelift-codegen code generator library" license = "Apache-2.0 WITH LLVM-exception" repository = "https://github.com/bytecodealliance/wasmtime" @@ -17,8 +17,8 @@ rustdoc-args = ["--document-private-items"] [dependencies] cranelift-srcgen = { workspace = true } -cranelift-assembler-x64-meta = { path = "../../assembler-x64/meta", version = "0.135.0" } -cranelift-codegen-shared = { path = "../shared", version = "0.135.0" } +cranelift-assembler-x64-meta = { path = "../../assembler-x64/meta", version = "0.135.1" } +cranelift-codegen-shared = { path = "../shared", version = "0.135.1" } pulley-interpreter = { workspace = true, optional = true } heck = "0.5.0" diff --git a/cranelift/codegen/meta/src/shared/settings.rs b/cranelift/codegen/meta/src/shared/settings.rs index bbbd4c96b694..fa4a2a122dfe 100644 --- a/cranelift/codegen/meta/src/shared/settings.rs +++ b/cranelift/codegen/meta/src/shared/settings.rs @@ -120,6 +120,19 @@ pub(crate) fn define() -> SettingGroup { false, ); + settings.add_bool( + "enable_nixe_abi", + "Enable the Nixe leaf-fragment ABI.", + r#" + Experimental Nixe backend integration for Linux x86-64 and AArch64. + Requires enable_pinned_reg. The pinned register points to a 64-byte-aligned + 16 KiB frame; the first 2 KiB are reserved for boundary transfers. + Fragments have no system-ABI arguments, calls, returns or stack frame. + Unsupported operations are rejected, not lowered through the system ABI. + "#, + false, + ); + settings.add_enum( "tls_model", "Defines the model used to perform TLS accesses.", diff --git a/cranelift/codegen/shared/Cargo.toml b/cranelift/codegen/shared/Cargo.toml index 0d059c967141..6a56c8c8b993 100644 --- a/cranelift/codegen/shared/Cargo.toml +++ b/cranelift/codegen/shared/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-codegen-shared" -version = "0.135.0" +version = "0.135.1" description = "For code shared between cranelift-codegen-meta and cranelift-codegen" license = "Apache-2.0 WITH LLVM-exception" repository = "https://github.com/bytecodealliance/wasmtime" diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rs index 9532b39c6ea6..f07ab3216230 100644 --- a/cranelift/codegen/src/alias_analysis.rs +++ b/cranelift/codegen/src/alias_analysis.rs @@ -38,10 +38,10 @@ //! To get this must-alias property, we compute a sparse table of //! "memory values": these are known equivalences between SSA `Value`s //! and particular locations in memory. The memory-values table is a -//! mapping from (last store, address expression, type) to SSA -//! value. At a store, we can insert into this table directly. At a -//! load, we can also insert, if we don't already have a value (from -//! the store that produced the load's value). +//! mapping from a memory location (address, type, byte order, etc...) +//! to a known value. At a store, we can insert into this table +//! directly. At a load, we can also insert, if we don't already have a +//! value (from the store that produced the load's value). //! //! Then we do a few optimizations at once given this table: //! @@ -74,14 +74,15 @@ //! up front, in `AliasAnalysis::observed_stores`, rather than tracking //! it as part of the per-block `LastStores` state. -use crate::cursor::CursorPosition; use crate::{FxHashMap, FxHashSet}; use crate::{ - cursor::{Cursor, FuncCursor}, + cursor::{Cursor, CursorPosition, FuncCursor}, dominator_tree::DominatorTree, flowgraph::ControlFlowGraph, inst_predicates::{inst_addr_offset_type, inst_store_data, visit_block_succs}, - ir::{AliasRegion, Block, Function, Inst, Opcode, Type, Value, immediates::Offset32}, + ir::{ + AliasRegion, Block, Endianness, Function, Inst, Opcode, Type, Value, immediates::Offset32, + }, post_dominator_tree::PostDominatorTree, trace, }; @@ -144,6 +145,48 @@ fn alias_regions_observed(func: &Function, inst: Inst, opcode: Opcode) -> AliasR } } +/// Who was the observer of some store instruction? +/// +/// `Option` -- where `None` is logically represented by the absense +/// of an entry in `AliasAnalysis::observed_stores` -- forms the following +/// lattice: +/// +/// ```ignore +/// None +/// / | \ \ \ +/// / | \ \ \ +/// / | \ \ \ +/// / | \ \ \ +/// inst0 inst1 instN... +/// \ | / / / +/// \ | / / / +/// \ | / / / +/// \ | / / / +/// Many +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Observer { + /// There was exactly one observer: this instruction. + One(Inst), + /// There were many observers. + Many, +} + +impl Observer { + fn meet(a: Self, b: Self) -> Self { + match (a, b) { + (Observer::Many, _) | (_, Observer::Many) => Observer::Many, + (Observer::One(a), Observer::One(b)) => { + if a == b { + Observer::One(a) + } else { + Observer::Many + } + } + } + } +} + /// For a given program point, the last-store instruction for each disjoint /// category of abstract state. /// @@ -176,14 +219,23 @@ pub struct LastStores { } /// Mark the store, if any, in the given last-store slot as observed. -fn observe(func: &Function, observed_stores: &mut FxHashSet, last_store: PackedOption) { - if let Some(inst) = last_store.expand() { +fn observe( + func: &Function, + observed_stores: &mut FxHashMap, + last_store: PackedOption, + observer: Inst, +) { + if let Some(last_store) = last_store.expand() { // NB: last-store slots do not always hold stores; they can also hold // calls, fences, and the markers that `LastStores::meet_from` inserts // where two control-flow paths disagree. Only actual stores can be DSE // candidates, so don't bother recording other instructions as observed. - if func.dfg.insts[inst].opcode().can_store() { - observed_stores.insert(inst); + if func.dfg.insts[last_store].opcode().can_store() { + let entry = observed_stores + .entry(last_store) + .or_insert(Observer::One(observer)); + *entry = Observer::meet(*entry, Observer::One(observer)); + trace!(" observed_stores[{last_store:?}] = {entry:?}"); } } } @@ -193,7 +245,7 @@ impl LastStores { &mut self, func: &Function, inst: Inst, - observed_stores: &mut FxHashSet, + observed_stores: &mut FxHashMap, ) { let opcode = func.dfg.insts[inst].opcode(); @@ -207,7 +259,7 @@ impl LastStores { // state of memory on trap. We do this by marking every last-store as // observed, but not clearing our last-store information. else if opcode.can_trap() { - self.observe_others(func, observed_stores, None); + self.observe_others(func, observed_stores, None, inst); } // Store instructions: update the last-store information for this // instruction's alias region, or, if it has no alias region, treat it @@ -216,30 +268,7 @@ impl LastStores { if let Some(memflags) = func.dfg.insts[inst].memflags() { match func.dfg.mem_flags[memflags].alias_region() { Some(region) => { - // NB: The old last-store instruction is *not* observed - // here, even though this new store instruction may not - // fully overwrite it. First, a new store in a block - // does not itself observe an old store in the same - // block. Second, the old store will never be an - // optimization candidate again from here on out: - // - // * We won't consider it again as we process the rest - // of this block, as it won't be in the last-store - // slot anymore. - // - // * What if we re-process this block in our initial - // fixed point loop? That implies this block is a - // member of a cycle in the CFG, but `meet_from` only - // propagates a store instruction when all - // predecessors agree on the same last-store - // instruction, but the predecessors already won't - // agree it is the old store since this block (which - // is on that path and therefore some kind of - // transitive predecessor) has already overridden it. - // - // Therefore, marking the old last-store as observed - // here is unnecessary (and, in fact, doing so would - // only inhibit optimization). + observe(func, observed_stores, self.regions[region], inst); self.regions[region] = inst.into(); // If this store can trap, then we need to observe @@ -282,9 +311,9 @@ impl LastStores { // incorrectly store to `v4+16`, when we otherwise // wouldn't have. if func.dfg.mem_flags[memflags].trap_code().is_some() { - self.observe_others(func, observed_stores, Some(region)); + self.observe_others(func, observed_stores, Some(region), inst); } else { - self.observe_trapping_others(func, observed_stores, region); + self.observe_trapping_others(func, observed_stores, region, inst); } } None => { @@ -303,15 +332,22 @@ impl LastStores { // instruction observes. else { match alias_regions_observed(func, inst, opcode) { - AliasRegionsObserved::All => self.observe_others(func, observed_stores, None), + AliasRegionsObserved::All => self.observe_others(func, observed_stores, None, inst), AliasRegionsObserved::Just(region) => { - observe(func, observed_stores, self.last_store_for_region(region)); + observe( + func, + observed_stores, + self.last_store_for_region(region), + inst, + ); // NB: Because stores without regions may alias any other // region, we have also observed the last such store, which // `self.last_fence` tracks. - observe(func, observed_stores, self.last_fence); + observe(func, observed_stores, self.last_fence, inst); + } + AliasRegionsObserved::Other => { + observe(func, observed_stores, self.last_fence, inst) } - AliasRegionsObserved::Other => observe(func, observed_stores, self.last_fence), AliasRegionsObserved::None => {} } } @@ -322,15 +358,16 @@ impl LastStores { fn observe_others( &self, func: &Function, - observed_stores: &mut FxHashSet, + observed_stores: &mut FxHashMap, excluding: Option, + observer: Inst, ) { for (region, last_store) in self.regions.iter() { if excluding.is_none_or(|r| r != region) { - observe(func, observed_stores, *last_store); + observe(func, observed_stores, *last_store, observer); } } - observe(func, observed_stores, self.last_fence); + observe(func, observed_stores, self.last_fence, observer); } /// Mark the last store to every region whose last store can trap, except for @@ -338,8 +375,9 @@ impl LastStores { fn observe_trapping_others( &self, func: &Function, - observed_stores: &mut FxHashSet, + observed_stores: &mut FxHashMap, excluding: AliasRegion, + observer: Inst, ) { let can_trap = |last_store: PackedOption| { last_store @@ -349,21 +387,26 @@ impl LastStores { for (region, last_store) in self.regions.iter() { if region != excluding && can_trap(*last_store) { - observe(func, observed_stores, *last_store); + observe(func, observed_stores, *last_store, observer); } } if can_trap(self.last_fence) { - observe(func, observed_stores, self.last_fence); + observe(func, observed_stores, self.last_fence, observer); } } /// Handle memory fence-like instructions by clearing all analysis data. - fn fence(&mut self, func: &Function, inst: Inst, observed_stores: &mut FxHashSet) { + fn fence( + &mut self, + func: &Function, + inst: Inst, + observed_stores: &mut FxHashMap, + ) { // A fence can observe every region, so every store we are currently // tracking for a region becomes observed. for (_region, last_store) in self.regions.iter() { - observe(func, observed_stores, *last_store); + observe(func, observed_stores, *last_store, inst); } self.regions.clear(); @@ -383,6 +426,43 @@ impl LastStores { } } + /// Get the contents of `inst`'s own alias region's slot, without falling + /// back to the last fence. + /// + /// Returns `None` when `inst` has no alias region. + fn raw_region_slot(&self, func: &Function, inst: Inst) -> Option> { + let region = func.dfg.insts[inst].alias_region(&func.dfg)?; + Some(self.regions[region]) + } + + /// Roll this state back to the memory version from just before `dead`, + /// which is a store being removed from the function by dead-store + /// elimination. + /// + /// `prev_region_slot` must be what `dead`'s own alias-region slot held + /// immediately before `dead` overwrote it, as recorded by `region_slot` + /// when `dead` itself was processed (that is, it must not be the last-fence + /// fallback). + /// + /// Only `dead`'s own alias-region slot is restored. A store with no alias + /// region is treated as a fence by `update`, which clears *every* region + /// slot, and we do not undo that; in that case, we leave this state + /// alone. Similarly, stores marked observed while processing `dead` stay + /// observed. + fn undo_store(&mut self, func: &Function, dead: Inst, prev_region_slot: PackedOption) { + debug_assert!(func.dfg.insts[dead].opcode().can_store()); + + let Some(region) = func.dfg.insts[dead].alias_region(&func.dfg) else { + return; + }; + + // Only roll back if `dead` really is the current last store to its + // region. + if self.regions[region].expand() == Some(dead) { + self.regions[region] = prev_region_slot; + } + } + /// Get the last-store instruction for the given `inst`'s alias region, if /// any. fn get_last_store(&self, func: &Function, inst: Inst) -> PackedOption { @@ -409,7 +489,7 @@ impl LastStores { func: &Function, rhs: &LastStores, loc: Inst, - observed_stores: &mut FxHashSet, + observed_stores: &mut FxHashMap, ) -> bool { // NB: Destructure to make sure we don't accidentally forget a // field. @@ -418,7 +498,7 @@ impl LastStores { last_fence, } = self; - let meet = |observed_stores: &mut FxHashSet, + let meet = |observed_stores: &mut FxHashMap, a: &mut PackedOption, b: PackedOption| -> bool { @@ -433,8 +513,8 @@ impl LastStores { // mark them both observed here. This keeps the // observed-stores set sound in the presence of loops and // control-flow join points. - observe(func, observed_stores, x.filter(|x| *x != loc).into()); - observe(func, observed_stores, y.filter(|y| *y != loc).into()); + observe(func, observed_stores, x.filter(|x| *x != loc).into(), loc); + observe(func, observed_stores, y.filter(|y| *y != loc).into(), loc); Some(loc) } }; @@ -464,8 +544,9 @@ impl LastStores { /// instruction to touch the disjoint category of abstract state we're /// accessing); (ii) the address must be the same (here ensured by /// having the same SSA value, which doesn't change after computed); -/// (iii) the offset must be the same; and (iv) the accessed type and -/// extension mode (e.g., 8-to-32, signed) must be the same. +/// (iii) the offset must be the same; (iv) the accessed type and +/// extension mode (e.g., 8-to-32, signed) must be the same; and (v) +/// the byte order of the two accesses must be the same. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct MemoryLoc { last_store: PackedOption, @@ -484,6 +565,46 @@ struct MemoryLoc { /// in place of extending loads when we know the memory value, but /// we haven't yet done this. extending_opcode: Option, + /// The byte order of this access, as explicitly specified in its memory + /// flags, or `None` when the access uses the target's native byte order. + /// + /// Without this, two accesses to the same address that disagree about byte + /// order would share a key, and we would happily forward a value from one + /// to the other, dropping the byte swap that the mismatch implies. + /// + /// We only record the *explicit* byte order here, rather than resolving + /// `None` to the target's native byte order. This keeps the pass + /// independent of the target, but does mean we never share a key between an + /// access that spells out the native byte order and one that leaves it + /// implicit. That scenario leads to missed optimizations, never + /// miscompiles, and is exceedingly rare, so we deem the trade off worth it + /// for simplicity. + endianness: Option, +} + +/// What is known to be in memory at an associated `MemoryLoc`. +#[derive(Clone, Copy, Debug)] +struct KnownValue { + /// The value held at the associated `MemoryLoc`. + value: Value, + + /// The instruction that produced `value`: either the load that read it out + /// of memory or the store that wrote it there. + /// + /// Kept around for quick dominance checks. + def_inst: Inst, + + /// When this entry was created by a store to a particular alias region, + /// whatever that region's last-store slot held just *before* `def_inst` + /// overwrote it, as given by `LastStores::region_slot`. + /// + /// `None` means either the entry was created by a load or by a store with + /// no alias region. Neither will ever undo `LastStores` state. + /// + /// `Some(maybe_inst)` contains the alias region slot's previous value, so + /// that it can be restored if `def_inst` is a dead store that gets + /// eliminated. + prev_region_slot: Option>, } /// The result of processing an instruction through alias analysis. @@ -527,28 +648,25 @@ pub struct AliasAnalysis<'a> { /// Unlike the last-store state in `block_input`, this is *not* flow /// sensitive: a store is either observable somewhere in the function or it /// is not. - observed_stores: FxHashSet, + observed_stores: FxHashMap, /// Input state to a basic block. block_input: FxHashMap, /// Known memory-value equivalences. This is the result of the - /// analysis. This is a mapping from (last store, address - /// expression, offset, type) to SSA `Value`. - /// - /// We keep the defining inst around for quick dominance checks. - mem_values: FxHashMap, + /// analysis. This is a mapping from a memory location to its known value. + mem_values: FxHashMap, } impl<'a> AliasAnalysis<'a> { /// Perform an alias analysis pass. pub fn new(func: &Function, domtree: &'a DominatorTree) -> AliasAnalysis<'a> { - trace!("alias analysis: input is:\n{:?}", func); + trace!("alias analysis input is:\n{func:?}"); assert!(domtree.is_valid()); let mut analysis = AliasAnalysis { domtree, post_dom_tree: None, - observed_stores: FxHashSet::default(), + observed_stores: FxHashMap::default(), block_input: FxHashMap::default(), mem_values: FxHashMap::default(), }; @@ -604,15 +722,13 @@ impl<'a> AliasAnalysis<'a> { .or_insert_with(|| LastStores::default()) .clone(); - trace!( - "alias analysis: input to block{} is {:?}", - block.index(), - state - ); + trace!("analyzing {block:?}"); + trace!(" initial block state = {state:?}"); for inst in func.layout.block_insts(block) { + trace!(" analyzing {inst:?}: {}", func.dfg.display_inst(inst)); state.update(func, inst, &mut self.observed_stores); - trace!("after inst{}: state is {:?}", inst.index(), state); + trace!(" updated state = {state:?}"); } visit_block_succs(func, block, |_inst, succ, _from_table| { @@ -635,6 +751,8 @@ impl<'a> AliasAnalysis<'a> { } }); } + + trace!("final observed_stores = {:#?}", self.observed_stores); } /// Get the starting state for a block. @@ -673,8 +791,9 @@ impl<'a> AliasAnalysis<'a> { // Check whether this store makes the last store dead. if let Some(last_store) = last_store.expand() { - // A store can only be dead when unobserved. - if !self.observed_stores.contains(&last_store) + // A store can only be dead when unobserved or only observed + // by its overwriter. + if self.observed_stores.get(&last_store).is_none_or(|o| *o == Observer::One(inst)) // This instruction doesn't make the last // store dead if it itself is the last store. && inst != last_store @@ -705,14 +824,51 @@ impl<'a> AliasAnalysis<'a> { // layout, so drop its entry from `mem_values`. This // maintains the invariant that `mem_values` only ever // references instructions that are still in the layout. + // + // NB: the entry we are looking for was keyed on the + // *dead* store's byte order, which `fully_overwrites` + // does not require to match this store's byte order. + // Everything else in the key does have to match, so we + // can take it from this store. let dead_loc = MemoryLoc { last_store: last_store.into(), address, offset, ty, extending_opcode: get_ext_opcode(opcode), + endianness: get_endianness(func, last_store), }; - self.mem_values.remove(&dead_loc); + let dead_entry = self.mem_values.remove(&dead_loc); + + // Roll our last-store state back to the memory version + // just before the dead store, so that `state` describes + // memory as if the dead store had never happened. + // + // Our callers remove the dead store from the layout and + // then reprocess this overwriting store. Without the + // rollback, that reprocessing keys its `mem_values` + // lookup on the instruction we just removed, finds + // nothing, and so fails to notice that the overwriter + // has now become an idempotent store. Chains like + // + // v1 = load.i32 region0 v0 + // store region0 v2, v0 ;; dead + // store region0 v1, v0 ;; idempotent, once the + // ;; dead store is gone + // + // would then need a whole additional pass over the + // function to collapse each link. + // + // A missing entry means we have no previous version to + // roll back to, and simply don't: either we never + // processed the dead store as a store in this pass (it + // can come from a precomputed `block_input` snapshot, + // for a predecessor block we have not walked yet) or it + // has no alias region and therefore no slot of its own. + if let Some(prev) = dead_entry.and_then(|e| e.prev_region_slot) { + state.undo_store(func, last_store, prev); + } + return OptResult::DeadStore { dead: last_store, overwriter: inst, @@ -726,8 +882,14 @@ impl<'a> AliasAnalysis<'a> { offset, ty, extending_opcode: get_ext_opcode(opcode), + endianness: get_endianness(func, inst), }; - if let Some((def_inst, known_value)) = self.mem_values.get(&check_loc).cloned() { + if let Some(KnownValue { + def_inst, + value: known_value, + .. + }) = self.mem_values.get(&check_loc).cloned() + { // Check for idempotent stores, where we are // storing the exact same value back to a location // that already has that value. @@ -742,8 +904,13 @@ impl<'a> AliasAnalysis<'a> { // We are removing this idempotent store in favor of the // original, so if this idempotent store was observed, // then the original must now be observed as well. - if self.observed_stores.contains(&inst) { - observe(func, &mut self.observed_stores, last_store); + if let Some(last_store) = last_store.expand() { + if let Some(observer) = self.observed_stores.get(&inst).copied() { + let entry = + self.observed_stores.entry(last_store).or_insert(observer); + *entry = Observer::meet(*entry, observer); + trace!(" observed_stores[{last_store:?}] = {entry:?}"); + } } return OptResult::IdempotentStore; @@ -757,9 +924,21 @@ impl<'a> AliasAnalysis<'a> { offset, ty, extending_opcode: get_ext_opcode(opcode), + endianness: get_endianness(func, inst), }; trace!(" --> updating known values in memory: {mem_loc:?} = {store_data}"); - self.mem_values.insert(mem_loc, (inst, store_data)); + self.mem_values.insert( + mem_loc, + KnownValue { + def_inst: inst, + value: store_data, + // NB: we use the raw region slot, without the + // last-fence fallback, because we don't want to move an + // instruction without a region into a region slot on + // DSE rollback. + prev_region_slot: state.raw_region_slot(func, inst), + }, + ); OptResult::None } else if opcode.can_load() { @@ -771,6 +950,7 @@ impl<'a> AliasAnalysis<'a> { offset, ty, extending_opcode: get_ext_opcode(opcode), + endianness: get_endianness(func, inst), }; trace!(" load with last_store at loc {mem_loc:?}"); @@ -784,8 +964,9 @@ impl<'a> AliasAnalysis<'a> { // load (stores will always dominate though if // their `last_store` survives through // meet-points to this use-site). - let aliased = if let Some((def_inst, value)) = - self.mem_values.get(&mem_loc).cloned() + let aliased = if let Some(KnownValue { + def_inst, value, .. + }) = self.mem_values.get(&mem_loc).cloned() { trace!(" see known value {value} from {def_inst}"); if self.domtree.dominates(def_inst, inst, &func.layout) { @@ -804,7 +985,16 @@ impl<'a> AliasAnalysis<'a> { // as a new equivalent value. if aliased.is_none() { trace!(" --> inserting load result {load_result} at loc {mem_loc:?}"); - self.mem_values.insert(mem_loc, (inst, load_result)); + self.mem_values.insert( + mem_loc, + KnownValue { + def_inst: inst, + value: load_result, + // A load does not advance the memory version, so + // there is no previous version to roll back to. + prev_region_slot: None, + }, + ); } match aliased { @@ -877,6 +1067,14 @@ impl<'a> AliasAnalysis<'a> { } } +/// Get the explicitly-specified byte order of the given memory access, +/// if any. +fn get_endianness(func: &Function, inst: Inst) -> Option { + func.dfg.insts[inst] + .memflags_data(&func.dfg) + .and_then(|flags| flags.explicit_endianness()) +} + fn get_ext_opcode(op: Opcode) -> Option { debug_assert!(op.can_load() || op.can_store()); match op { @@ -941,6 +1139,12 @@ fn fully_overwrites( return false; } + // NB: unlike store-to-load forwarding and redundant-load + // elimination, our two stores' byte orders do *not* have to match + // Both write the same range of bytes, just in a different order + // within that range, and the overwriting store's bytes are the + // ones that survive either way. + // Both must write the same address, offset, and type. match inst_addr_offset_type(func, maybe_dead) { Some((addr, offset, ty)) => { diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index 7d6026c97be0..b8bf16fcf437 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -184,6 +184,7 @@ impl Context { self.verify_if(isa)?; + crate::nixe::validate_entries(&self.func, isa)?; self.compute_cfg(); self.compute_domtree(); self.eliminate_unreachable_code(isa)?; diff --git a/cranelift/codegen/src/egraph/elaborate.rs b/cranelift/codegen/src/egraph/elaborate.rs index 86443e5cb126..aa3e00a29793 100644 --- a/cranelift/codegen/src/egraph/elaborate.rs +++ b/cranelift/codegen/src/egraph/elaborate.rs @@ -559,7 +559,7 @@ impl<'a> Elaborator<'a> { // placing too much register pressure on the entire // function. This is modeled with the `.saturating_sub(1)` // as the default if there's otherwise no maximum. - let loop_hoist_level = arg_values + let mut loop_hoist_level = arg_values .iter() .map(|&value| { // Find the outermost loop level at which @@ -581,6 +581,17 @@ impl<'a> Elaborator<'a> { }) .max() .unwrap_or(self.loop_stack.len().saturating_sub(1)); + // The Nixe root exists only for analysis, not execution. + // Keep LICM within executable dominators: nested loops can + // still hoist into outer loops or canonical ingress blocks. + if !self.func.nixe_entries.is_empty() { + while loop_hoist_level < self.loop_stack.len() + && Some(self.loop_stack[loop_hoist_level].hoist_block) + == self.func.layout.entry_block() + { + loop_hoist_level += 1; + } + } trace!( " -> loop hoist level: {:?}; cur loop depth: {:?}, loop_stack: {:?}", loop_hoist_level, diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index a194246ad852..f1ac008cfc9a 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -201,6 +201,11 @@ pub struct FunctionStencil { /// call instructions. pub debug_tags: DebugTags, + /// Nixe canonical external entries, in caller order. Set with + /// `nixe::set_entries`; the layout's first block is then analysis-only. + /// Entries have no block parameters and define their own live inputs. + pub nixe_entries: alloc::vec::Vec, + /// An optional global value which represents an expression evaluating to /// the stack limit for this function. This `GlobalValue` will be /// interpreted in the prologue, if necessary, to insert a stack check to @@ -219,6 +224,7 @@ impl FunctionStencil { self.layout.clear(); self.srclocs.clear(); self.debug_tags.clear(); + self.nixe_entries.clear(); self.stack_limit = None; } @@ -420,6 +426,7 @@ impl Function { srclocs: SecondaryMap::new(), stack_limit: None, debug_tags: DebugTags::default(), + nixe_entries: alloc::vec::Vec::new(), }, params: FunctionParameters::new(), } diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 806ad1566a92..a8e24f823f90 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -556,6 +556,20 @@ impl ABIMachineSpec for AArch64MachineDeps { spilltmp_reg() } + fn nixe_frame_reg() -> Reg { + xreg(21) + } + + fn gen_nixe_frame_addr(offset: u32, into_reg: Writable) -> Inst { + Inst::LoadAddr { + rd: into_reg, + mem: AMode::RegOffset { + rn: xreg(21), + off: i64::from(offset), + }, + } + } + fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, ty: Type) -> Inst { let mem = AMode::RegOffset { rn: base, @@ -1177,11 +1191,15 @@ impl ABIMachineSpec for AArch64MachineDeps { } fn get_machine_env(flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { + if flags.enable_nixe_abi() { + static MACHINE_ENV: MachineEnv = create_reg_env(true, true); + return &MACHINE_ENV; + } if flags.enable_pinned_reg() { - static MACHINE_ENV: MachineEnv = create_reg_env(true); + static MACHINE_ENV: MachineEnv = create_reg_env(true, false); &MACHINE_ENV } else { - static MACHINE_ENV: MachineEnv = create_reg_env(false); + static MACHINE_ENV: MachineEnv = create_reg_env(false, false); &MACHINE_ENV } } @@ -1640,7 +1658,7 @@ const WINCH_CLOBBERS: PRegSet = winch_clobbers(); const ALL_CLOBBERS: PRegSet = all_clobbers(); const NO_CLOBBERS: PRegSet = PRegSet::empty(); -const fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { +const fn create_reg_env(enable_pinned_reg: bool, nixe: bool) -> MachineEnv { const fn preg(r: Reg) -> PReg { r.to_real_reg().unwrap().preg() } @@ -1701,8 +1719,6 @@ const fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { ], non_preferred_regs_by_class: [ PRegSet::empty() - .with(preg(xreg(19))) - .with(preg(xreg(20))) // x21 is pinned reg if enabled; we add to this list below if not. .with(preg(xreg(22))) .with(preg(xreg(23))) @@ -1727,6 +1743,11 @@ const fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { scratch_by_class: [None, None, None], }; + if !nixe { + env.non_preferred_regs_by_class[0] = env.non_preferred_regs_by_class[0] + .with(preg(xreg(19))) + .with(preg(xreg(20))); + } if !enable_pinned_reg { debug_assert!(PINNED_REG == 21); env.non_preferred_regs_by_class[0].add(preg(xreg(PINNED_REG))); @@ -1734,3 +1755,27 @@ const fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { env } + +#[cfg(test)] +mod nixe_tests { + use super::*; + #[test] + fn nixe_register_pool_preserves_ordinary_allocation_policy() { + let nixe = create_reg_env(true, true); + let normal = create_reg_env(false, false); + for hw in 0..32 { + let reg = PReg::new(hw, RegClass::Int); + let contains = |env: &MachineEnv| { + env.preferred_regs_by_class[0].contains(reg) + || env.non_preferred_regs_by_class[0].contains(reg) + }; + assert_eq!( + contains(&nixe), + contains(&normal) && ![19, 20, 21].contains(&hw) + ); + if [16, 17].contains(&hw) { + assert!(!contains(&nixe)); + } + } + } +} diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 6bae128cecf8..f877b1d264d5 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -3997,9 +3997,12 @@ (if-let new_offset (i32_checked_add x offset)) (amode_no_more_iconst ty y new_offset)) +(decl abi_slot_amode (i32) AMode) +(extern constructor abi_slot_amode abi_slot_amode) + (rule 3 (amode ty (stack_addr _ slot offset1) offset2) - (AMode.SlotOffset + (abi_slot_amode (abi_stackslot_offset_into_slot_region slot offset1 offset2))) (attr amode_no_more_iconst (veri chain)) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index d50940b2ee8a..1ecdb3d56e29 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -24,8 +24,8 @@ use crate::{ ValueList, immediates::*, types::*, }, isa::aarch64::abi::AArch64MachineDeps, - isa::aarch64::inst::SImm7Scaled, isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm}, + isa::aarch64::inst::{AMode, SImm7Scaled, xreg}, machinst::{ CallArgList, CallRetList, InstOutput, MachInst, VCodeConstant, VCodeConstantData, abi::ArgPair, ty_bits, @@ -74,6 +74,18 @@ pub struct ExtendedValue { } impl Context for IsleContext<'_, '_, MInst, AArch64Backend> { + fn abi_slot_amode(&mut self, offset: i32) -> AMode { + if self.backend.flags.enable_nixe_abi() { + AMode::RegOffset { + rn: xreg(21), + off: i64::from(offset) + i64::from(crate::nixe::TRANSFER_BYTES), + } + } else { + AMode::SlotOffset { + off: i64::from(offset), + } + } + } isle_lower_prelude_methods!(); fn gen_call_info( diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 81e87f8e350a..ffc80ba50a69 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -507,6 +507,17 @@ impl ABIMachineSpec for X64ABIMachineSpec { regs::r10() } + fn nixe_frame_reg() -> Reg { + regs::r15() + } + + fn gen_nixe_frame_addr(offset: u32, into_reg: Writable) -> Self::I { + let mem = SyntheticAmode::Real(Amode::imm_reg(offset.try_into().unwrap(), regs::r15())); + Inst::External { + inst: asm::inst::leaq_rm::new(into_reg, mem).into(), + } + } + fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, ty: Type) -> Self::I { // Only ever used for I64s, F128s and vectors; if that changes, see if // the ExtKind below needs to be changed. @@ -873,11 +884,15 @@ impl ABIMachineSpec for X64ABIMachineSpec { } fn get_machine_env(flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { + if flags.enable_nixe_abi() { + static MACHINE_ENV: MachineEnv = create_reg_env_systemv(true, true); + return &MACHINE_ENV; + } if flags.enable_pinned_reg() { - static MACHINE_ENV: MachineEnv = create_reg_env_systemv(true); + static MACHINE_ENV: MachineEnv = create_reg_env_systemv(true, false); &MACHINE_ENV } else { - static MACHINE_ENV: MachineEnv = create_reg_env_systemv(false); + static MACHINE_ENV: MachineEnv = create_reg_env_systemv(false, false); &MACHINE_ENV } } @@ -1277,7 +1292,7 @@ const fn all_clobbers() -> PRegSet { .with(regs::fpr_preg(XMM15)) } -const fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { +const fn create_reg_env_systemv(enable_pinned_reg: bool, nixe: bool) -> MachineEnv { const fn preg(r: Reg) -> PReg { r.to_real_reg().unwrap().preg() } @@ -1293,8 +1308,7 @@ const fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { .with(preg(regs::rdx())) .with(preg(regs::r8())) .with(preg(regs::r9())) - .with(preg(regs::r10())) - .with(preg(regs::r11())), + .with(preg(regs::r10())), // Preferred XMMs: the first 8, which can have smaller encodings // with AVX instructions. PRegSet::empty() @@ -1313,9 +1327,7 @@ const fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { // Non-preferred GPRs: callee-saved in the SysV ABI. PRegSet::empty() .with(preg(regs::rbx())) - .with(preg(regs::r12())) - .with(preg(regs::r13())) - .with(preg(regs::r14())), + .with(preg(regs::r12())), // Non-preferred XMMs: the last 8 registers, which can have larger // encodings with AVX instructions. PRegSet::empty() @@ -1334,6 +1346,12 @@ const fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { scratch_by_class: [None, None, None], }; + if !nixe { + env.preferred_regs_by_class[0] = env.preferred_regs_by_class[0].with(preg(regs::r11())); + env.non_preferred_regs_by_class[0] = env.non_preferred_regs_by_class[0] + .with(preg(regs::r13())) + .with(preg(regs::r14())); + } debug_assert!(regs::PINNED_REG == cranelift_assembler_x64::gpr::enc::R15); if !enable_pinned_reg { env.non_preferred_regs_by_class[0] = @@ -1346,6 +1364,22 @@ const fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { #[cfg(test)] mod tests { use super::*; + #[test] + fn nixe_register_pool_preserves_ordinary_allocation_policy() { + let nixe = create_reg_env_systemv(true, true); + let normal = create_reg_env_systemv(false, false); + for hw in 0..16 { + let reg = PReg::new(hw, RegClass::Int); + let contains = |env: &MachineEnv| { + env.preferred_regs_by_class[0].contains(reg) + || env.non_preferred_regs_by_class[0].contains(reg) + }; + assert_eq!( + contains(&nixe), + contains(&normal) && ![11, 13, 14, 15].contains(&hw) + ); + } + } use crate::machinst::abi::Callee; use alloc::vec::Vec; diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 8c04e7f89369..b5d15a8e00b6 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -738,6 +738,14 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> { #[inline] fn synthetic_amode_slot(&mut self, offset: i32) -> SyntheticAmode { + if self.backend.flags.enable_nixe_abi() { + return SyntheticAmode::Real(Amode::imm_reg( + offset + .checked_add(crate::nixe::TRANSFER_BYTES as i32) + .unwrap(), + regs::r15(), + )); + } SyntheticAmode::SlotOffset { simm32: offset } } diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 24446b83699c..b9344dbfff10 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -60,6 +60,7 @@ pub mod inline; pub mod ir; pub mod isa; pub mod loop_analysis; +pub mod nixe; pub mod post_dominator_tree; pub mod print_errors; pub mod settings; diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index c7b42116b5d6..2428762fe604 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -458,6 +458,17 @@ pub trait ABIMachineSpec { /// SP-based offset). fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable) -> Self::I; + /// Fixed external frame base, available only on the Nixe target ISAs. + fn nixe_frame_reg() -> Reg { + unreachable!("Nixe ABI was not validated for this ISA") + } + + /// Compute an external frame address without referring to the host stack. + fn gen_nixe_frame_addr(offset: u32, into_reg: Writable) -> Self::I { + let _ = (offset, into_reg); + unreachable!("Nixe ABI was not validated for this ISA") + } + /// Get a fixed register to use to compute a stack limit. This is needed for /// certain sequences generated after the register allocator has already /// run. This must satisfy two requirements: @@ -1217,6 +1228,7 @@ impl Callee { isa_flags: &M::F, sigs: &SigSet, ) -> CodegenResult { + crate::nixe::validate(f, isa)?; trace!("ABI: func signature {:?}", f.signature); let flags = isa.flags().clone(); @@ -2172,6 +2184,12 @@ impl Callee { // Offset from beginning of stackslot area. let stack_off = self.sized_stackslots[slot] as i64; let sp_off: i64 = stack_off + (offset as i64); + if self.flags.enable_nixe_abi() { + return M::gen_nixe_frame_addr( + crate::nixe::TRANSFER_BYTES + u32::try_from(sp_off).unwrap(), + into_reg, + ); + } M::gen_get_stack_addr(StackAMode::Slot(sp_off), into_reg) } @@ -2212,8 +2230,32 @@ impl Callee { function_calls: FunctionCalls, ) -> CodegenResult<()> { let bytes = M::word_bytes(); - let total_stacksize = self.stackslots_size + bytes * spillslots as u32; let mask = M::stack_align(self.call_conv) - 1; + if self.flags.enable_nixe_abi() { + let total_stacksize = u32::try_from(spillslots) + .ok() + .and_then(|slots| slots.checked_mul(bytes)) + .and_then(|bytes| self.stackslots_size.checked_add(bytes)) + .and_then(|bytes| checked_round_up(bytes, mask)) + .ok_or(CodegenError::ImplLimitExceeded)?; + if function_calls != FunctionCalls::None || self.outgoing_args_size != 0 { + return Err(CodegenError::Unsupported( + "Nixe ABI: backend-introduced calls are not supported".into(), + )); + } + if total_stacksize > crate::nixe::FRAME_BYTES - crate::nixe::TRANSFER_BYTES { + return Err(CodegenError::ImplLimitExceeded); + } + self.frame_layout = Some(FrameLayout { + word_bytes: bytes, + fixed_frame_storage_size: total_stacksize, + stackslots_size: self.stackslots_size, + function_calls, + ..FrameLayout::default() + }); + return Ok(()); + } + let total_stacksize = self.stackslots_size + bytes * spillslots as u32; let total_stacksize = (total_stacksize + mask) & !mask; // 16-align the stack. let frame_layout = M::compute_frame_layout( self.call_conv, @@ -2255,6 +2297,9 @@ impl Callee { /// This should include any stack frame or other setup necessary to use the /// other methods (`load_arg`, `store_retval`, and spillslot accesses.) pub fn gen_prologue(&self) -> SmallInstVec { + if self.flags.enable_nixe_abi() { + return smallvec![]; + } let frame_layout = self.frame_layout(); let mut insts = smallvec![]; @@ -2326,6 +2371,10 @@ impl Callee { /// emitting this in the lowering logic), because the epilogue code comes /// before the return and the two are likely closely related. pub fn gen_epilogue(&self) -> SmallInstVec { + assert!( + !self.flags.enable_nixe_abi(), + "Nixe fragments cannot return" + ); let frame_layout = self.frame_layout(); let mut insts = smallvec![]; @@ -2421,6 +2470,15 @@ impl Callee { let sp_off = self.get_spillslot_offset(to_slot); trace!("gen_spill: {from_reg:?} into slot {to_slot:?} at offset {sp_off}"); + if self.flags.enable_nixe_abi() { + return M::gen_store_base_offset( + M::nixe_frame_reg(), + i32::try_from(i64::from(crate::nixe::TRANSFER_BYTES) + sp_off).unwrap(), + Reg::from(from_reg), + ty, + ); + } + let from = StackAMode::Slot(sp_off); ::gen_store_stack(from, Reg::from(from_reg), ty) } @@ -2433,6 +2491,15 @@ impl Callee { let sp_off = self.get_spillslot_offset(from_slot); trace!("gen_reload: {to_reg:?} from slot {from_slot:?} at offset {sp_off}"); + if self.flags.enable_nixe_abi() { + return M::gen_load_base_offset( + to_reg.map(Reg::from), + M::nixe_frame_reg(), + i32::try_from(i64::from(crate::nixe::TRANSFER_BYTES) + sp_off).unwrap(), + ty, + ); + } + let from = StackAMode::Slot(sp_off); ::gen_load_stack(from, to_reg.map(Reg::from), ty) } @@ -2444,9 +2511,14 @@ impl Callee { /// set up by compiled code in stackslots allocated for that /// purpose. pub fn frame_slot_metadata(&self) -> MachBufferFrameLayout { - let frame_to_fp_offset = self.sp_to_fp_offset(); + let nixe = self.flags.enable_nixe_abi(); + let frame_to_fp_offset = if nixe { 0 } else { self.sp_to_fp_offset() }; let mut stackslots = SecondaryMap::with_capacity(self.sized_stackslots.len()); - let storage_area_base = self.frame_layout().outgoing_args_size; + let storage_area_base = if nixe { + crate::nixe::TRANSFER_BYTES + } else { + self.frame_layout().outgoing_args_size + }; for (slot, storage_area_offset) in &self.sized_stackslots { stackslots[slot] = MachBufferStackSlot { offset: storage_area_base.checked_add(*storage_area_offset).unwrap(), @@ -2456,6 +2528,9 @@ impl Callee { MachBufferFrameLayout { frame_to_fp_offset, stackslots, + nixe_frame_size: nixe.then_some( + crate::nixe::TRANSFER_BYTES + self.frame_layout().fixed_frame_storage_size, + ), } } } diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs index f19720e6c29a..6c9e8d8cd8fc 100644 --- a/cranelift/codegen/src/machinst/blockorder.rs +++ b/cranelift/codegen/src/machinst/blockorder.rs @@ -70,6 +70,8 @@ use crate::{machinst::*, trace}; /// Mapping from CLIF BBs to VCode BBs. #[derive(Debug)] pub struct BlockLoweringOrder { + nixe_root: Option, + nixe_entries: Vec, /// Lowered blocks, in BlockIndex order. Each block is some combination of /// (i) a CLIF block, and (ii) inserted crit-edge blocks before or after; /// see [LoweredBlock] for details. @@ -298,6 +300,8 @@ impl BlockLoweringOrder { })); let result = BlockLoweringOrder { + nixe_root: (!f.nixe_entries.is_empty()).then(|| f.layout.entry_block().unwrap()), + nixe_entries: f.nixe_entries.clone(), lowered_order, lowered_succ_indices, lowered_succ_ranges, @@ -315,6 +319,19 @@ impl BlockLoweringOrder { &self.lowered_order[..] } + /// Root and outgoing edge blocks used only for multi-entry analysis. + pub fn is_nixe_analysis_block(&self, block: BlockIndex) -> bool { + match (self.nixe_root, self.lowered_order[block.index()]) { + (Some(root), LoweredBlock::Orig { block }) => root == block, + (Some(root), LoweredBlock::CriticalEdge { pred, .. }) => root == pred, + _ => false, + } + } + + pub fn nixe_entries(&self) -> &[Block] { + &self.nixe_entries + } + /// Get the BlockIndex, if any, for a given Block. /// /// The result will be `None` if the given Block is unreachable @@ -339,6 +356,9 @@ impl BlockLoweringOrder { /// target. pub fn is_indirect_branch_target(&self, block: BlockIndex) -> bool { self.indirect_branch_targets.contains(&block) + || self.lowered_order[block.index()] + .orig_block() + .is_some_and(|b| self.nixe_entries.contains(&b)) } } diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index fba4ee3d81dc..0ec5ab529d73 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -360,6 +360,7 @@ pub struct MachBuffer { /// containing a function body, this allows interpretation of /// runtime state given a view of an active stack frame. frame_layout: Option, + nixe_entries: Vec<(ir::Block, CodeOffset)>, } impl MachBufferFinalized { @@ -383,6 +384,7 @@ impl MachBufferFinalized { unwind_info: self.unwind_info, alignment: self.alignment, frame_layout: self.frame_layout, + nixe_entries: self.nixe_entries, nop_units: self.nop_units, } } @@ -425,6 +427,8 @@ pub struct MachBufferFinalized { /// containing a function body, this allows interpretation of /// runtime state given a view of an active stack frame. pub(crate) frame_layout: Option, + /// Selected canonical entry offsets, including entry allocator edits. + pub nixe_entries: Vec<(ir::Block, CodeOffset)>, /// Any unwind info at a given location. pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, /// The required alignment of this buffer. @@ -530,6 +534,7 @@ impl MachBuffer { used_constants: Default::default(), open_patchable: false, frame_layout: None, + nixe_entries: Vec::new(), } } @@ -1678,6 +1683,7 @@ impl MachBuffer { alignment, frame_layout: self.frame_layout, nop_units: I::gen_nop_units(), + nixe_entries: self.nixe_entries, } } @@ -1871,6 +1877,10 @@ impl MachBuffer { debug_assert!(self.frame_layout.is_none()); self.frame_layout = Some(frame_layout); } + + pub(crate) fn set_nixe_entries(&mut self, entries: Vec<(ir::Block, CodeOffset)>) { + self.nixe_entries = entries; + } } impl Extend for MachBuffer { @@ -2350,6 +2360,9 @@ impl MachBranch { derive(serde_derive::Serialize, serde_derive::Deserialize) )] pub struct MachBufferFrameLayout { + /// Nixe external-frame extent, including transfer storage, when enabled. + /// Stack-slot offsets then refer to the pinned base, not SP or FP. + pub nixe_frame_size: Option, /// Offset from bottom of frame to FP (near top of frame). This /// allows reading the frame given only FP. pub frame_to_fp_offset: u32, diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index 2efb29a90579..f812cf5ca4d1 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -746,7 +746,52 @@ impl VCode { let _tt = timing::vcode_emit(); let mut buffer = MachBuffer::new(); buffer.set_log2_min_function_alignment(self.log2_min_function_alignment); - let mut bb_starts: Vec> = vec![]; + // Index by VCode block, not emission order: cold blocks move and the + // Nixe analysis root is omitted altogether. + let mut bb_starts: Vec> = if flags.machine_code_cfg_info() { + vec![None; self.num_blocks()] + } else { + vec![] + }; + + // Lowering must not reintroduce a live-in defined in omitted code. + // Check VRegs, not allocated physical registers (which are reused). + if !self.block_order.nixe_entries().is_empty() { + let mut omitted_defs = crate::FxHashSet::default(); + for block in 0..self.num_blocks() { + if self + .block_order + .is_nixe_analysis_block(BlockIndex::new(block)) + { + for inst in self.block_ranges.get(block) { + for operand in &self.operands[self.operand_ranges.get(inst)] { + if operand.kind() == OperandKind::Def { + omitted_defs.insert(operand.vreg()); + } + } + } + } + } + for block in 0..self.num_blocks() { + if !self + .block_order + .is_nixe_analysis_block(BlockIndex::new(block)) + { + for inst in self.block_ranges.get(block) { + if self.operands[self.operand_ranges.get(inst)] + .iter() + .any(|op| { + op.kind() == OperandKind::Use && omitted_defs.contains(&op.vreg()) + }) + { + return Err(CodegenError::Unsupported( + "Nixe entry depends on omitted machine code".into(), + )); + } + } + } + } + } // The first M MachLabels are reserved for block indices. buffer.reserve_labels_for_blocks(self.num_blocks()); @@ -821,6 +866,9 @@ impl VCode { let mut total_bb_padding = 0; for &block in final_order.iter() { + if self.block_order.is_nixe_analysis_block(block) { + continue; + } trace!("emitting block {:?}", block); // Call the new block hook for state @@ -880,14 +928,13 @@ impl VCode { // branch opts, note that the removed blocks were removed. let cur_offset = buffer.cur_offset(); if last_offset.is_some() && cur_offset <= last_offset.unwrap() { - for i in (0..bb_starts.len()).rev() { - if bb_starts[i].is_some() && cur_offset > bb_starts[i].unwrap() { - break; + for start in &mut bb_starts { + if start.is_some_and(|offset| offset >= cur_offset) { + *start = None; } - bb_starts[i] = None; } } - bb_starts.push(Some(cur_offset)); + bb_starts[block.index()] = Some(cur_offset); last_offset = Some(cur_offset); } @@ -1157,12 +1204,31 @@ impl VCode { } } + // Consumers such as disassembly traverse physical code order, even + // though the bookkeeping above is indexed by logical VCode block. + bb_offsets.sort_unstable(); self.monotonize_inst_offsets(&mut inst_offsets[..], func_body_len); let value_labels_ranges = self.compute_value_labels_ranges(regalloc, &inst_offsets[..], func_body_len); // Store metadata about frame layout in the MachBuffer. buffer.set_frame_layout(self.abi.frame_slot_metadata()); + buffer.set_nixe_entries( + self.block_order + .nixe_entries() + .iter() + .map(|&block| { + let index = self + .block_order + .lowered_index_for_block(block) + .expect("validated external entry"); + ( + block, + buffer.resolve_label_offset(MachLabel::from_block(index)), + ) + }) + .collect(), + ); Ok(EmitResult { buffer: buffer.finish(&self.constants, ctrl_plane), diff --git a/cranelift/codegen/src/nixe.rs b/cranelift/codegen/src/nixe.rs new file mode 100644 index 000000000000..3655c9147422 --- /dev/null +++ b/cranelift/codegen/src/nixe.rs @@ -0,0 +1,569 @@ +//! Initial, opt-in Nixe leaf-fragment ABI. +//! +//! This is not a system calling convention. The caller owns every register +//! and places a 64-byte-aligned spill area at offset zero of NativeFrame, +//! addressed by r15 (x86-64) or x21 (AArch64). Canonical external entries define +//! their own inputs. Production gateways, physical fast-entry contracts and +//! boundary state maps are not implemented here yet. + +use crate::{CodegenError, CodegenResult, ir, isa::TargetIsa}; +use alloc::format; + +/// Bytes reserved for boundary transfers; never allocated to backend spills. +pub const TRANSFER_BYTES: u32 = 2048; +/// Total fixed storage, including boundary transfers. +pub const FRAME_BYTES: u32 = 16384; + +/// Declare canonical external entries into a single compiled unit. +/// +/// Each entry defines its own inputs (no block parameters). An analysis-only +/// root makes every entry reachable to dominance and regalloc; it and its +/// outgoing critical edges are never emitted. The selector has no runtime +/// meaning. Final offsets are exported by `MachBufferFinalized::nixe_entries`. +/// Call once after constructing the body, before optimization. +pub fn set_entries(func: &mut ir::Function, entries: &[ir::Block]) -> CodegenResult<()> { + use crate::cursor::{Cursor, FuncCursor}; + use ir::{InstBuilder, types}; + let fail = || CodegenError::Unsupported("Nixe ABI: invalid canonical entries".into()); + if entries.is_empty() || !func.nixe_entries.is_empty() { + return Err(fail()); + } + for (index, &block) in entries.iter().enumerate() { + if !func.layout.is_block_inserted(block) + || !func.dfg.block_params(block).is_empty() + || entries[..index].contains(&block) + { + return Err(fail()); + } + } + let root = func.dfg.make_block(); + let first = func.layout.entry_block().unwrap(); + func.layout.insert_block(root, first); + let targets: alloc::vec::Vec<_> = entries + .iter() + .map(|&block| func.dfg.block_call(block, &[])) + .collect(); + let table = func.create_jump_table(ir::JumpTableData::new(targets[0], &targets[1..])); + let mut c = FuncCursor::new(func).at_bottom(root); + let selector = c.ins().get_pinned_reg(types::I64); + let selector = c.ins().ireduce(types::I32, selector); + c.ins().br_table(selector, table); + c.func.nixe_entries.extend_from_slice(entries); + Ok(()) +} + +/// Validate the analysis-root boundary whenever optimization/lowering consumes +/// it. No computation defined there may become an external entry's live-in. +pub(crate) fn validate_entries(func: &ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> { + if func.nixe_entries.is_empty() { + return Ok(()); + } + let fail = |detail| CodegenError::Unsupported(format!("Nixe entries: {detail}")); + if !isa.flags().enable_nixe_abi() { + return Err(fail("enable_nixe_abi is required")); + } + let root = func + .layout + .entry_block() + .ok_or_else(|| fail("missing analysis root"))?; + for &entry in &func.nixe_entries { + if entry == root + || !func.layout.is_block_inserted(entry) + || !func.dfg.block_params(entry).is_empty() + { + return Err(fail("entries must exist and define their own inputs")); + } + } + for block in func.layout.blocks() { + for inst in func.layout.block_insts(block) { + if block == root + && !matches!( + func.dfg.insts[inst].opcode(), + ir::Opcode::GetPinnedReg + | ir::Opcode::Ireduce + | ir::Opcode::BrTable + | ir::Opcode::Jump + ) + { + return Err(fail("unexpected computation in analysis root")); + } + for dest in func.dfg.insts[inst] + .branch_destination(&func.dfg.jump_tables, &func.dfg.exception_tables) + { + let dest = dest.block(&func.dfg.value_lists); + if dest == root || (block == root && !func.nixe_entries.contains(&dest)) { + return Err(fail("invalid analysis-root edge")); + } + } + if block != root { + for value in func.dfg.inst_values(inst) { + if let ir::ValueDef::Result(def, _) = func.dfg.value_def(value) { + if func.layout.inst_block(def) == Some(root) { + return Err(fail("entry depends on an analysis-root value")); + } + } + } + } + } + } + let last = func + .layout + .last_inst(root) + .ok_or_else(|| fail("empty analysis root"))?; + let destinations = + func.dfg.insts[last].branch_destination(&func.dfg.jump_tables, &func.dfg.exception_tables); + if func.nixe_entries.iter().any(|entry| { + !destinations + .iter() + .any(|dest| dest.block(&func.dfg.value_lists) == *entry) + }) { + return Err(fail("analysis root must reach every external entry")); + } + Ok(()) +} + +#[cfg(all(test, feature = "x86", feature = "arm64"))] +mod multi_entry; + +pub(crate) fn validate(func: &ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> { + validate_entries(func, isa)?; + if !isa.flags().enable_nixe_abi() { + return Ok(()); + } + let unsupported = |detail: &str| CodegenError::Unsupported(format!("Nixe ABI: {detail}")); + if !matches!(isa.name(), "x64" | "aarch64") + || isa.triple().operating_system != target_lexicon::OperatingSystem::Linux + { + return Err(unsupported("only Linux x86-64 and AArch64 are supported")); + } + if !isa.flags().enable_pinned_reg() { + return Err(unsupported("enable_pinned_reg is required")); + } + if !func.signature.params.is_empty() || !func.signature.returns.is_empty() { + return Err(unsupported( + "system-ABI parameters and results are not supported", + )); + } + if func.stack_limit.is_some() || !func.dynamic_stack_slots.is_empty() { + return Err(unsupported( + "stack limits and dynamic stack slots are not supported", + )); + } + if func + .dfg + .values_labels + .as_ref() + .is_some_and(|labels| !labels.is_empty()) + { + return Err(unsupported( + "debug value locations are not Nixe physical state maps", + )); + } + for slot in func.sized_stack_slots.values() { + if slot.align_shift > 6 || slot.size > FRAME_BYTES - TRANSFER_BYTES { + return Err(unsupported( + "explicit stack slot exceeds fixed frame bounds", + )); + } + } + for block in func.layout.blocks() { + for inst in func.layout.block_insts(block) { + let op = func.dfg.insts[inst].opcode(); + if op.is_call() + || op.is_return() + || matches!( + op, + ir::Opcode::SetPinnedReg + | ir::Opcode::GetFramePointer + | ir::Opcode::GetReturnAddress + | ir::Opcode::GetStackPointer + | ir::Opcode::StackSwitch + | ir::Opcode::TlsValue + ) + { + return Err(unsupported(&format!( + "{op} requires an unimplemented native boundary" + ))); + } + } + } + Ok(()) +} + +#[cfg(all(test, feature = "x86", feature = "arm64"))] +mod tests { + use super::*; + use crate::cursor::{Cursor, FuncCursor}; + use crate::ir::{ + InstBuilder, MemFlagsData as MemFlags, StackSlotData, StackSlotKind, TrapCode, types, + }; + use crate::settings::{self, Configurable}; + use crate::{Context, isa}; + use alloc::{string::String, vec::Vec}; + use cranelift_control::ControlPlane; + + pub(super) fn target( + triple: &str, + allocator: &str, + nixe: bool, + ) -> alloc::sync::Arc { + let mut flags = settings::builder(); + flags.set("enable_pinned_reg", "true").unwrap(); + flags + .set("enable_nixe_abi", if nixe { "true" } else { "false" }) + .unwrap(); + flags.set("regalloc_algorithm", allocator).unwrap(); + flags.set("machine_code_cfg_info", "true").unwrap(); + flags + .set( + "opt_level", + if allocator == "single_pass" { + "none" + } else { + "speed" + }, + ) + .unwrap(); + isa::lookup(triple.parse().unwrap()) + .unwrap() + .finish(settings::Flags::new(flags)) + .unwrap() + } + + fn fragment(count: usize, slot_bytes: u32) -> ir::Function { + let mut f = ir::Function::new(); + let slot = f.create_sized_stack_slot(StackSlotData::new( + StackSlotKind::ExplicitSlot, + slot_bytes, + 4, + )); + let block = f.dfg.make_block(); + f.layout.append_block(block); + let mut c = FuncCursor::new(&mut f).at_bottom(block); + let frame = c.ins().get_pinned_reg(types::I64); + let data = c.ins().load(types::I64, MemFlags::trusted(), frame, 0); + let mut ints = Vec::new(); + let mut vectors = Vec::new(); + for i in 0..count { + ints.push( + c.ins() + .load(types::I64, MemFlags::new(), data, (i * 8) as i32), + ); + vectors.push( + c.ins() + .load(types::I64X2, MemFlags::new(), data, (4096 + i * 16) as i32), + ); + } + for i in (0..count).rev() { + c.ins() + .store(MemFlags::new(), ints[i], data, (8192 + i * 8) as i32); + c.ins() + .store(MemFlags::new(), vectors[i], data, (12288 + i * 16) as i32); + } + let addr = c.ins().stack_addr(types::I64, slot, 0); + c.ins().store(MemFlags::trusted(), data, addr, 0); + // Also make the slot address escape, exercising LEA/LoadAddr instead + // of only the folded stack-slot memory operand. + c.ins().store(MemFlags::new(), addr, data, 2048); + c.ins().trap(TrapCode::unwrap_user(1)); + f + } + + pub(super) fn compile( + f: ir::Function, + isa: &dyn TargetIsa, + ) -> Result { + let mut cx = Context::for_function(f); + cx.set_disasm(true); + cx.compile(isa, &mut ControlPlane::default()) + .map_err(|e| format!("{e:?}"))?; + Ok(cx.take_compiled_code().unwrap()) + } + + #[test] + fn pressure_uses_external_frame_on_both_targets_and_allocators() { + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + for allocator in ["single_pass", "backtracking"] { + let isa = target(triple, allocator, true); + for count in [40, 57, 72] { + let code = compile(fragment(count, 32), &*isa).unwrap(); + let frame = code.buffer.frame_layout().unwrap(); + let extent = frame.nixe_frame_size.unwrap(); + assert!(extent > TRANSFER_BYTES + 32, "pressure must actually spill"); + assert!(extent <= FRAME_BYTES); + for slot in frame.stackslots.values() { + assert!(slot.offset >= TRANSFER_BYTES && slot.offset + 32 <= extent); + } + #[cfg(feature = "disas")] + let native_asm = code.disassemble(None, &isa.to_capstone().unwrap()).unwrap(); + #[cfg(feature = "disas")] + let asm = &native_asm; + #[cfg(not(feature = "disas"))] + let asm = code.vcode.as_ref().unwrap(); + let forbidden: &[&str] = if triple.starts_with("x86") { + &[ + "rsp", "rbp", "esp", "ebp", "r11", "r11d", "r13", "r13d", "r14", + "r14d", "ret", "call", "push", "pop", + ] + } else { + &[ + "sp", "x29", "w29", "x19", "w19", "x20", "w20", "ret", "bl", "blr", + ] + }; + for token in asm.split(|c: char| !c.is_ascii_alphanumeric()) { + assert!( + !forbidden.contains(&token), + "{triple}/{allocator}: {token}\n{asm}" + ); + } + assert!( + asm.contains(if triple.starts_with("x86") { + "r15" + } else { + "x21" + }), + "{asm}" + ); + } + } + } + } + + #[test] + fn oversized_frames_and_system_returns_are_rejected() { + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + for allocator in ["single_pass", "backtracking"] { + let isa = target(triple, allocator, true); + let full = compile(fragment(0, FRAME_BYTES - TRANSFER_BYTES), &*isa).unwrap(); + assert_eq!( + full.buffer.frame_layout().unwrap().nixe_frame_size, + Some(FRAME_BYTES) + ); + assert!(compile(fragment(72, FRAME_BYTES - TRANSFER_BYTES), &*isa).is_err()); + let mut f = ir::Function::new(); + let block = f.dfg.make_block(); + f.layout.append_block(block); + FuncCursor::new(&mut f).at_bottom(block).ins().return_(&[]); + assert!( + compile(f.clone(), &*isa) + .unwrap_err() + .contains("unimplemented native boundary") + ); + // The new mode must not suppress ordinary ABI prologues/returns. + let ordinary = target(triple, allocator, false); + assert!( + compile(f, &*ordinary) + .unwrap() + .vcode + .unwrap() + .contains("ret") + ); + } + } + } + + #[test] + fn invalid_configuration_and_calls_fail_before_producing_code() { + let mut flags = settings::builder(); + flags.set("enable_nixe_abi", "true").unwrap(); + let isa = isa::lookup("x86_64-unknown-linux-gnu".parse().unwrap()) + .unwrap() + .finish(settings::Flags::new(flags)) + .unwrap(); + assert!( + compile(fragment(0, 16), &*isa) + .unwrap_err() + .contains("enable_pinned_reg") + ); + + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + let isa = target(triple, "backtracking", true); + let mut f = fragment(0, 16); + f.signature.params.push(ir::AbiParam::new(types::I64)); + let block = f.layout.entry_block().unwrap(); + f.dfg.append_block_param(block, types::I64); + assert!(compile(f, &*isa).unwrap_err().contains("parameters")); + let mut f = fragment(0, 16); + let end = f.layout.last_inst(f.layout.entry_block().unwrap()).unwrap(); + let sig = f.import_signature(ir::Signature::new(isa::CallConv::SystemV)); + let mut c = FuncCursor::new(&mut f).at_inst(end); + let addr = c.ins().iconst(types::I64, 0); + c.ins().call_indirect(sig, addr, &[]); + assert!( + compile(f, &*isa) + .unwrap_err() + .contains("unimplemented native boundary") + ); + } + } + + #[test] + fn zero_byte_markers_preserve_selected_offsets_in_hot_and_cold_blocks() { + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + for allocator in ["single_pass", "backtracking"] { + let isa = target(triple, allocator, true); + let mut f = ir::Function::new(); + let root = f.dfg.make_block(); + let hot = f.dfg.make_block(); + let cold = f.dfg.make_block(); + for block in [root, hot, cold] { + f.layout.append_block(block); + } + f.layout.set_cold(cold); + let mut c = FuncCursor::new(&mut f).at_bottom(root); + let condition = c.ins().get_pinned_reg(types::I64); + c.ins().brif(condition, hot, &[], cold, &[]); + for (block, id) in [(hot, 11), (cold, 22)] { + let mut c = FuncCursor::new(&mut f).at_bottom(block); + let marker = c.ins().sequence_point(); + c.func.debug_tags.set(marker, [ir::DebugTag::User(id)]); + c.ins().trap(TrapCode::unwrap_user(id as u8)); + } + let code = compile(f, &*isa).unwrap(); + let tags: Vec<_> = code.buffer.debug_tags().collect(); + assert_eq!(tags.len(), 2); + for tag in tags { + let [ir::DebugTag::User(id)] = tag.tags else { + panic!("missing entry identity") + }; + assert!( + code.buffer + .traps() + .iter() + .any(|trap| trap.offset == tag.offset + && trap.code == TrapCode::unwrap_user(*id as u8)) + ); + } + } + } + } + + /// Probe regalloc's actual operand API, not its optional debug ranges. + /// The virtual root is analysis-only here: this test does not implement + /// Cranelift multi-entry lowering or demonstrate safe root removal. + #[test] + fn fixed_entry_defs_and_exact_boundary_allocations_need_no_new_allocator() { + use regalloc2::{ + Algorithm, Allocation, Block, Inst, InstRange, MachineEnv, Operand, PReg, PRegSet, + RegClass, RegallocOptions, VReg, + }; + const VALUES: usize = 12; + const STRIDE: usize = VALUES + 1; + struct Probe { + operands: Vec>, + children: [Block; 2], + root: [Block; 1], + } + impl regalloc2::Function for Probe { + fn num_insts(&self) -> usize { + self.operands.len() + } + fn num_blocks(&self) -> usize { + 3 + } + fn entry_block(&self) -> Block { + Block::new(0) + } + fn block_insns(&self, b: Block) -> InstRange { + let (start, end) = if b.index() == 0 { + (0, 1) + } else { + (1 + (b.index() - 1) * STRIDE, 1 + b.index() * STRIDE) + }; + InstRange::new(Inst::new(start), Inst::new(end)) + } + fn block_succs(&self, b: Block) -> &[Block] { + if b.index() == 0 { &self.children } else { &[] } + } + fn block_preds(&self, b: Block) -> &[Block] { + if b.index() == 0 { &[] } else { &self.root } + } + fn block_params(&self, _: Block) -> &[VReg] { + &[] + } + fn is_ret(&self, i: Inst) -> bool { + i.index() == STRIDE || i.index() == 2 * STRIDE + } + fn is_branch(&self, i: Inst) -> bool { + i.index() == 0 + } + fn branch_blockparams(&self, _: Block, _: Inst, _: usize) -> &[VReg] { + &[] + } + fn inst_operands(&self, i: Inst) -> &[Operand] { + &self.operands[i.index()] + } + fn inst_clobbers(&self, _: Inst) -> PRegSet { + PRegSet::empty() + } + fn num_vregs(&self) -> usize { + 2 * VALUES + } + fn spillslot_size(&self, _: RegClass) -> usize { + 1 + } + } + let mut probe = Probe { + operands: alloc::vec![alloc::vec![]], + children: [Block::new(1), Block::new(2)], + root: [Block::new(0)], + }; + for entry in 0..2 { + for v in 0..VALUES { + let vr = VReg::new(entry * VALUES + v, RegClass::Int); + let op = if v == 0 { + Operand::reg_fixed_def(vr, PReg::new(entry, RegClass::Int)) + } else { + Operand::reg_def(vr) + }; + probe.operands.push(alloc::vec![op]); + } + probe.operands.push( + (0..VALUES) + .map(|v| Operand::any_use(VReg::new(entry * VALUES + v, RegClass::Int))) + .collect(), + ); + } + let mut regs = PRegSet::empty(); + for reg in 0..4 { + regs.add(PReg::new(reg, RegClass::Int)); + } + let env = MachineEnv { + preferred_regs_by_class: [regs, PRegSet::empty(), PRegSet::empty()], + non_preferred_regs_by_class: [PRegSet::empty(); 3], + scratch_by_class: [None; 3], + fixed_stack_slots: alloc::vec![], + }; + for algorithm in [Algorithm::Ion, Algorithm::Fastalloc] { + let result = regalloc2::run( + &probe, + &env, + &RegallocOptions { + algorithm, + validate_ssa: true, + ..RegallocOptions::default() + }, + ) + .unwrap(); + let mut checker = regalloc2::checker::Checker::new(&probe, &env); + checker.prepare(&result); + checker.run().unwrap(); + for entry in 0..2 { + assert_eq!( + result.inst_allocs(Inst::new(1 + entry * STRIDE)), + &[Allocation::reg(PReg::new(entry, RegClass::Int))] + ); + let locations = result.inst_allocs(Inst::new((entry + 1) * STRIDE)); + assert_eq!(locations.len(), VALUES); + assert!(locations.iter().any(|l| l.as_stack().is_some())); + for loc in locations { + if let Some(slot) = loc.as_stack() { + assert!(TRANSFER_BYTES + (slot.index() as u32 + 1) * 8 <= FRAME_BYTES); + } else { + assert!(loc.as_reg().is_some()); + } + } + } + } + } +} diff --git a/cranelift/codegen/src/nixe/multi_entry.rs b/cranelift/codegen/src/nixe/multi_entry.rs new file mode 100644 index 000000000000..4209ca1297e4 --- /dev/null +++ b/cranelift/codegen/src/nixe/multi_entry.rs @@ -0,0 +1,523 @@ +//! Canonical multi-entry regression: one optimized body, actual block offsets, +//! and no emitted analysis root. The native adapter is test-only, not the final +//! stackless gateway. Loops cover the original LICM counterexample. + +use super::tests::{compile, target}; +use crate::cursor::{Cursor, FuncCursor}; +use crate::ir::{self, InstBuilder, MemFlagsData as MemFlags, TrapCode, types}; +use alloc::vec::Vec; + +const VALUES: usize = 24; +const ADDEND: i64 = 0x1234_5678_9abc_def0; +const VECTOR_ADDEND: [u64; 2] = [0xfedc_ba98_7654_3210, 0x1234_5678_9abc_def0]; + +fn marker(c: &mut FuncCursor<'_>, id: u32) { + let inst = c.ins().sequence_point(); + c.func.debug_tags.set(inst, [ir::DebugTag::User(id)]); +} + +fn fragment(cold: bool, looping: bool) -> ir::Function { + let mut f = ir::Function::new(); + let a = f.dfg.make_block(); + let b = f.dfg.make_block(); + let body = f.dfg.make_block(); + for block in [a, b, body] { + f.layout.append_block(block); + } + if cold { + f.layout.set_cold(b); + } + let ints: Vec<_> = (0..VALUES) + .map(|_| f.dfg.append_block_param(body, types::I64)) + .collect(); + let vectors: Vec<_> = (0..VALUES) + .map(|_| f.dfg.append_block_param(body, types::I64X2)) + .collect(); + for (block, entry) in [(a, 0), (b, 1)] { + let mut c = FuncCursor::new(&mut f).at_bottom(block); + marker(&mut c, entry + 1); + // Inputs are defined locally, never passed down from the virtual root. + let frame = c.ins().get_pinned_reg(types::I64); + let mut args: Vec = Vec::new(); + for i in 0..VALUES { + let value = c + .ins() + .load(types::I64, MemFlags::trusted(), frame, (i * 8) as i32); + args.push(c.ins().iadd_imm_s(value, ADDEND + entry as i64).into()); + } + let constant = if looping { + let bytes: Vec<_> = VECTOR_ADDEND.iter().flat_map(|v| v.to_le_bytes()).collect(); + let constant = c.func.dfg.constants.insert(ir::ConstantData::from(bytes)); + Some(c.ins().vconst(types::I64X2, constant)) + } else { + None + }; + for i in 0..VALUES { + let value = c.ins().load( + types::I64X2, + MemFlags::trusted(), + frame, + (256 + i * 16) as i32, + ); + let value = if let Some(constant) = constant { + c.ins().iadd(value, constant) + } else { + value + }; + args.push(value.into()); + } + if looping { + let remaining = c.ins().load(types::I64, MemFlags::trusted(), frame, 1808); + let remaining = c.ins().iadd_imm_s(remaining, -1); + c.ins().store(MemFlags::trusted(), remaining, frame, 1808); + c.ins().brif(remaining, block, &[], body, &args); + } else { + c.ins().jump(body, &args); + } + } + let mut c = FuncCursor::new(&mut f).at_bottom(body); + marker(&mut c, 3); + let frame = c.ins().get_pinned_reg(types::I64); + for i in (0..VALUES).rev() { + c.ins() + .store(MemFlags::trusted(), ints[i], frame, (768 + i * 8) as i32); + c.ins().store( + MemFlags::trusted(), + vectors[i], + frame, + (1024 + i * 16) as i32, + ); + } + c.ins().trap(TrapCode::unwrap_user(1)); + super::set_entries(&mut f, &[a, b]).unwrap(); + f +} + +fn offsets(code: &crate::CompiledCode) -> [usize; 3] { + let mut result = [usize::MAX; 3]; + for tag in code.buffer.debug_tags() { + let [ir::DebugTag::User(id @ 1..=3)] = tag.tags else { + panic!("unexpected tag") + }; + assert_eq!( + result[*id as usize - 1], + usize::MAX, + "duplicated body/entry" + ); + result[*id as usize - 1] = tag.offset as usize; + } + assert!( + result + .iter() + .all(|offset| *offset < code.code_buffer().len()) + ); + assert_ne!(result[0], result[1]); + assert_ne!(result[0], result[2]); + assert_ne!(result[1], result[2]); + assert_eq!(code.buffer.nixe_entries.len(), 2); + for (i, (_, offset)) in code.buffer.nixe_entries.iter().enumerate() { + assert!( + (*offset as usize) <= result[i], + "entry includes allocator edits before marker" + ); + result[i] = *offset as usize; + } + assert_eq!(result[0].min(result[1]), 0, "no root or dispatcher bytes"); + assert!( + !code + .vcode + .as_ref() + .unwrap() + .lines() + .any(|line| line == "block0:") + ); + result +} + +#[test] +fn optimized_loop_constant_is_not_initialized_in_the_virtual_root() { + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + let isa = target(triple, "backtracking", true); + let mut cx = crate::Context::for_function(fragment(false, true)); + cx.optimize(&*isa, &mut cranelift_control::ControlPlane::default()) + .unwrap(); + let root = cx.func.layout.entry_block().unwrap(); + assert!( + !cx.func + .layout + .block_insts(root) + .any(|inst| cx.func.dfg.insts[inst].opcode() == ir::Opcode::Vconst) + ); + } +} + +#[test] +fn licm_still_hoists_into_an_executable_preheader() { + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + let isa = target(triple, "backtracking", true); + let mut f = ir::Function::new(); + let entry = f.dfg.make_block(); + let header = f.dfg.make_block(); + let exit = f.dfg.make_block(); + for block in [entry, header, exit] { + f.layout.append_block(block); + } + FuncCursor::new(&mut f) + .at_bottom(entry) + .ins() + .jump(header, &[]); + let mut c = FuncCursor::new(&mut f).at_bottom(header); + let frame = c.ins().get_pinned_reg(types::I64); + let constant = c.func.dfg.constants.insert(ir::ConstantData::from( + VECTOR_ADDEND + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + )); + let vector = c.ins().vconst(types::I64X2, constant); + c.ins().store(MemFlags::trusted(), vector, frame, 16); + let counter = c.ins().load(types::I64, MemFlags::trusted(), frame, 0); + let counter = c.ins().iadd_imm_s(counter, -1); + c.ins().store(MemFlags::trusted(), counter, frame, 0); + c.ins().brif(counter, header, &[], exit, &[]); + FuncCursor::new(&mut f) + .at_bottom(exit) + .ins() + .trap(TrapCode::unwrap_user(1)); + super::set_entries(&mut f, &[entry]).unwrap(); + let mut cx = crate::Context::for_function(f); + cx.optimize(&*isa, &mut cranelift_control::ControlPlane::default()) + .unwrap(); + let has_constant = |block| { + cx.func + .layout + .block_insts(block) + .any(|inst| cx.func.dfg.insts[inst].opcode() == ir::Opcode::Vconst) + }; + assert!(has_constant(entry), "valid LICM must remain enabled"); + assert!(!has_constant(header)); + assert!(!has_constant(cx.func.layout.entry_block().unwrap())); + compile(cx.func, &*isa).unwrap(); + } +} + +#[test] +fn invalid_external_entry_boundaries_are_rejected() { + let isa = target("x86_64-unknown-linux-gnu", "backtracking", true); + let mut f = fragment(false, true); + assert!(super::set_entries(&mut f, &[]).is_err()); + let ordinary = target("x86_64-unknown-linux-gnu", "backtracking", false); + assert!( + compile(f.clone(), &*ordinary) + .unwrap_err() + .contains("enable_nixe_abi") + ); + let root = f.layout.entry_block().unwrap(); + let entry = f.nixe_entries[0]; + let root_value = f.dfg.inst_results(f.layout.first_inst(root).unwrap())[0]; + let first = f.layout.first_inst(entry).unwrap(); + // Even a valid SSA use of the analysis-only selector is not a native input. + FuncCursor::new(&mut f).at_inst(first).ins().store( + MemFlags::trusted(), + root_value, + root_value, + 0, + ); + assert!( + compile(f, &*isa) + .unwrap_err() + .contains("analysis-root value") + ); + + let mut f = fragment(false, false); + let root = f.layout.entry_block().unwrap(); + let end = f.layout.last_inst(root).unwrap(); + FuncCursor::new(&mut f).at_inst(end).ins().sequence_point(); + assert!( + compile(f, &*isa) + .unwrap_err() + .contains("unexpected computation") + ); +} + +fn independent_entries(count: usize) -> ir::Function { + let mut f = ir::Function::new(); + let mut entries = Vec::new(); + for i in 0..count { + let block = f.dfg.make_block(); + f.layout.append_block(block); + if i % 2 != 0 { + f.layout.set_cold(block); + } + entries.push(block); + let mut c = FuncCursor::new(&mut f).at_bottom(block); + let frame = c.ins().get_pinned_reg(types::I64); + let value = c.ins().iconst(types::I64, ADDEND + i as i64); + c.ins().store(MemFlags::trusted(), value, frame, 0); + c.ins().trap(TrapCode::unwrap_user(1)); + } + super::set_entries(&mut f, &entries).unwrap(); + f +} + +#[test] +fn one_and_many_entries_have_no_emitted_dispatcher() { + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + for allocator in ["single_pass", "backtracking"] { + let isa = target(triple, allocator, true); + for count in [1, 3, 8] { + let f = independent_entries(count); + let entries = f.nixe_entries.clone(); + let code = compile(f, &*isa).unwrap(); + assert_eq!( + code.buffer + .nixe_entries + .iter() + .map(|&(block, _)| block) + .collect::>(), + entries + ); + assert_eq!( + code.buffer + .nixe_entries + .iter() + .map(|&(_, offset)| offset) + .min(), + Some(0) + ); + assert!(code.bb_edges.is_empty(), "no root edges in the emitted CFG"); + assert_eq!(code.bb_starts.len(), count); + } + } + } +} + +#[test] +fn two_entries_and_one_shared_body_survive_both_compiler_profiles() { + for triple in ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] { + for allocator in ["single_pass", "backtracking"] { + for cold in [false, true] { + for looping in [false, true] { + let isa = target(triple, allocator, true); + let code = compile(fragment(cold, looping), &*isa).unwrap(); + offsets(&code); + assert!(code.bb_starts.windows(2).all(|pair| pair[0] < pair[1])); + #[cfg(feature = "disas")] + code.disassemble(None, &isa.to_capstone().unwrap()).unwrap(); + assert!(code.buffer.relocs().is_empty()); + assert!( + code.buffer.frame_layout().unwrap().nixe_frame_size.unwrap() + > super::TRANSFER_BYTES + ); + } + } + } + } +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::*; + use core::ffi::c_void; + + // Test-owned ABI adapter only. The generated leaf has no prologue, calls + // or returns; its terminal UD2 is replaced with RET in the executable test + // copy. This does NOT prove the final stackless gateway/exit protocol. + core::arch::global_asm!( + ".pushsection .text", + ".global nixe_probe_enter", + ".hidden nixe_probe_enter", + ".type nixe_probe_enter,@function", + "nixe_probe_enter:", + "push rbp", + "push rbx", + "push r12", + "push r13", + "push r14", + "push r15", + "sub rsp, 8", + "mov r15, rdi", + "mov r11, rsi", + "mov rax, rdx", + "mov rbx, rdx", + "mov rcx, rdx", + "mov rbp, rdx", + "mov rsi, rdx", + "mov rdi, rdx", + "mov r8, rdx", + "mov r9, rdx", + "mov r10, rdx", + "mov r12, rdx", + "mov r13, rdx", + "mov r14, rdx", + "pxor xmm0, xmm0", + "pxor xmm1, xmm1", + "pxor xmm2, xmm2", + "pxor xmm3, xmm3", + "pxor xmm4, xmm4", + "pxor xmm5, xmm5", + "pxor xmm6, xmm6", + "pxor xmm7, xmm7", + "pxor xmm8, xmm8", + "pxor xmm9, xmm9", + "pxor xmm10, xmm10", + "pxor xmm11, xmm11", + "pxor xmm12, xmm12", + "pxor xmm13, xmm13", + "pxor xmm14, xmm14", + "pxor xmm15, xmm15", + "call r11", + "add rsp, 8", + "pop r15", + "pop r14", + "pop r13", + "pop r12", + "pop rbx", + "pop rbp", + "ret", + ".size nixe_probe_enter, .-nixe_probe_enter", + ".popsection", + ); + + unsafe extern "C" { + fn nixe_probe_enter(frame: *mut u64, entry: *const u8, poison: u64); + fn mmap( + addr: *mut c_void, + len: usize, + prot: i32, + flags: i32, + fd: i32, + offset: i64, + ) -> *mut c_void; + fn mprotect(addr: *mut c_void, len: usize, prot: i32) -> i32; + fn munmap(addr: *mut c_void, len: usize) -> i32; + } + + struct Executable { + ptr: *mut c_void, + len: usize, + } + + impl Executable { + fn new(bytes: &[u8]) -> Self { + // Linux x86-64 constants: PROT_READ|WRITE, MAP_PRIVATE|ANONYMOUS. + // SAFETY: anonymous allocation, checked before copying; permissions + // become RX before execution and never RWX. No external relocations. + unsafe { + let ptr = mmap(core::ptr::null_mut(), bytes.len(), 3, 0x22, -1, 0); + assert_ne!(ptr as isize, -1, "mmap failed"); + let mapping = Self { + ptr, + len: bytes.len(), + }; + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.cast(), bytes.len()); + assert_eq!(mprotect(ptr, bytes.len(), 5), 0, "mprotect RX failed"); + mapping + } + } + } + + impl Drop for Executable { + fn drop(&mut self) { + // SAFETY: this mapping is owned and no execution outlives it. + unsafe { assert_eq!(munmap(self.ptr, self.len), 0) }; + } + } + + #[repr(C, align(64))] + struct Frame([u64; super::super::FRAME_BYTES as usize / 8]); + + #[test] + fn native_entries_execute_without_the_virtual_root() { + execute(false, &["single_pass", "backtracking"]); + } + + #[test] + fn native_loop_entries_execute_without_the_virtual_root() { + execute(true, &["single_pass", "backtracking"]); + } + + #[test] + fn native_one_and_many_entries_execute_by_block_identity() { + for allocator in ["single_pass", "backtracking"] { + let isa = target("x86_64-unknown-linux-gnu", allocator, true); + for count in [1, 3, 8] { + let code = compile(independent_entries(count), &*isa).unwrap(); + let mut bytes = code.code_buffer().to_vec(); + assert!(code.buffer.relocs().is_empty()); + for trap in code.buffer.traps() { + let offset = trap.offset as usize; + assert_eq!(&bytes[offset..offset + 2], &[0x0f, 0x0b]); + bytes[offset..offset + 2].copy_from_slice(&[0xc3, 0x90]); + } + let mapping = Executable::new(&bytes); + for (i, &(_, offset)) in code.buffer.nixe_entries.iter().enumerate() { + let mut frame = Frame([0; super::super::FRAME_BYTES as usize / 8]); + // SAFETY: same leaf fixture and SysV adapter as below. + unsafe { + nixe_probe_enter( + frame.0.as_mut_ptr(), + mapping.ptr.cast::().add(offset as usize), + u64::MAX, + ); + } + assert_eq!(frame.0[0], ADDEND as u64 + i as u64); + } + } + } + } + + fn execute(looping: bool, allocators: &[&str]) { + for allocator in allocators { + for cold in [false, true] { + let isa = target("x86_64-unknown-linux-gnu", allocator, true); + let code = compile(fragment(cold, looping), &*isa).unwrap(); + let [a, b, _] = offsets(&code); + let mut bytes = code.code_buffer().to_vec(); + assert!(code.buffer.relocs().is_empty()); + assert_eq!(code.buffer.traps().len(), 1); + let exit = code.buffer.traps()[0].offset as usize; + assert_eq!(&bytes[exit..exit + 2], &[0x0f, 0x0b]); + bytes[exit..exit + 2].copy_from_slice(&[0xc3, 0x90]); + let mapping = Executable::new(&bytes); + for (entry, offset) in [a, b].into_iter().enumerate() { + for seed in [0_u64, 0x7654_3210_fedc_ba98, u64::MAX] { + let mut frame = Frame([seed; super::super::FRAME_BYTES as usize / 8]); + for i in 0..VALUES { + frame.0[i] = seed.wrapping_add(i as u64 * 37); + frame.0[32 + i * 2] = seed ^ (i as u64 * 59); + frame.0[33 + i * 2] = !seed ^ (i as u64 * 83); + } + frame.0[226] = 3; + // SAFETY: this fixture uses bounded frame offsets and + // no helpers; the adapter preserves SysV callee-saves. + unsafe { + nixe_probe_enter( + frame.0.as_mut_ptr(), + mapping.ptr.cast::().add(offset), + !seed, + ); + } + for i in 0..VALUES { + assert_eq!( + frame.0[96 + i], + frame.0[i].wrapping_add(ADDEND as u64 + entry as u64), + "{allocator}, cold={cold}, entry={entry}, value={i}" + ); + for lane in 0..2 { + let expected = frame.0[32 + i * 2 + lane] + .wrapping_add(if looping { VECTOR_ADDEND[lane] } else { 0 }); + assert_eq!( + frame.0[128 + i * 2 + lane], + expected, + "{allocator}, cold={cold}, entry={entry}, vector={i}, lane={lane}" + ); + } + } + assert_eq!(frame.0[226], if looping { 0 } else { 3 }); + } + } + } + } + } +} diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs index b16f45c43e53..6cf93b919ae5 100644 --- a/cranelift/codegen/src/settings.rs +++ b/cranelift/codegen/src/settings.rs @@ -504,6 +504,7 @@ is_pic = false use_colocated_libcalls = false enable_nan_canonicalization = false enable_pinned_reg = false +enable_nixe_abi = false enable_llvm_abi_extensions = false enable_multi_ret_implicit_sret = false unwind_info = true diff --git a/cranelift/control/Cargo.toml b/cranelift/control/Cargo.toml index 8362db6f5be0..633ca1e72a66 100644 --- a/cranelift/control/Cargo.toml +++ b/cranelift/control/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-control" -version = "0.135.0" +version = "0.135.1" description = "White-box fuzz testing framework" license = "Apache-2.0 WITH LLVM-exception" repository = "https://github.com/bytecodealliance/wasmtime" diff --git a/cranelift/entity/Cargo.toml b/cranelift/entity/Cargo.toml index 14396e22f6ba..9e6a589d50eb 100644 --- a/cranelift/entity/Cargo.toml +++ b/cranelift/entity/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-entity" -version = "0.135.0" +version = "0.135.1" description = "Data structures using entity references as mapping keys" license = "Apache-2.0 WITH LLVM-exception" documentation = "https://docs.rs/cranelift-entity" diff --git a/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif b/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif index af926b0a0bf8..08188dd75cb8 100644 --- a/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif +++ b/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif @@ -21,7 +21,7 @@ block0(v0: i64, v1: i32): ; block0(v0: i64, v1: i32): ; v2 = load.i64 notrap aligned region0 v0 ; trapz v2, user42 -; store notrap aligned region0 v2, v0 ; v4 = iadd v1, v1 ; return v4 ; } + diff --git a/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif b/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif new file mode 100644 index 000000000000..02e5ecd5b8a5 --- /dev/null +++ b/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif @@ -0,0 +1,181 @@ +test optimize precise-output +set opt_level=speed +target x86_64 + +;; Removing a dead store must expose the *previous* memory version to the store +;; that overwrote it, so that a save/clear/restore sequence collapses entirely in +;; a single pass rather than one link per pass. +function %save_clear_restore(i64) { + region0 = 0 "flags" +block0(v0: i64): + v1 = load.i32 notrap aligned region0 v0 + v2 = iconst.i32 0 + store notrap aligned region0 v2, v0 + store notrap aligned region0 v1, v0 + return +} + +; function %save_clear_restore(i64) fast { +; region0 = 0 "flags" +; +; block0(v0: i64): +; v1 = load.i32 notrap aligned region0 v0 +; return +; } + +;; The same, but with several dead stores between the load and the restore. +function %save_clobber_many_restore(i64, i32, i32) { + region0 = 0 "flags" +block0(v0: i64, v1: i32, v2: i32): + v3 = load.i32 notrap aligned region0 v0 + store notrap aligned region0 v1, v0 + store notrap aligned region0 v2, v0 + store notrap aligned region0 v1, v0 + store notrap aligned region0 v3, v0 + return +} + +; function %save_clobber_many_restore(i64, i32, i32) fast { +; region0 = 0 "flags" +; +; block0(v0: i64, v1: i32, v2: i32): +; v3 = load.i32 notrap aligned region0 v0 +; return +; } + +;; Two independent flags, each in its own alias region, are both collapsed. +;; +;; Note that the accesses are interleaved: unwinding one region's dead store +;; must not disturb the other region's last-store state. +function %two_regions_interleaved(i64, i64) { + region0 = 0 "flags0" + region1 = 1 "flags1" +block0(v0: i64, v1: i64): + v2 = load.i32 notrap aligned region0 v0 + v3 = load.i32 notrap aligned region1 v1 + v4 = iconst.i32 0 + store notrap aligned region0 v4, v0 + store notrap aligned region1 v4, v1 + store notrap aligned region0 v2, v0 + store notrap aligned region1 v3, v1 + return +} + +; function %two_regions_interleaved(i64, i64) fast { +; region0 = 0 "flags0" +; region1 = 1 "flags1" +; +; block0(v0: i64, v1: i64): +; v2 = load.i32 notrap aligned region0 v0 +; v3 = load.i32 notrap aligned region1 v1 +; return +; } + +;; The restore is folded across intervening blocks, so long as nothing in them +;; observes the flag. +function %save_clear_restore_cross_block(i64) { + region0 = 0 "flags" +block0(v0: i64): + v1 = load.i32 notrap aligned region0 v0 + v2 = iconst.i32 0 + store notrap aligned region0 v2, v0 + jump block1 + +block1: + jump block2 + +block2: + store notrap aligned region0 v1, v0 + return +} + +; function %save_clear_restore_cross_block(i64) fast { +; region0 = 0 "flags" +; +; block0(v0: i64): +; v1 = load.i32 notrap aligned region0 v0 +; jump block1 +; +; block1: +; jump block2 +; +; block2: +; return +; } + +;; Negative test: a call between the clear and the restore observes the cleared +;; flag, so neither store may be removed. +function %call_observes_cleared_flag(i64) { + region0 = 0 "flags" + fn0 = %g(i64) +block0(v0: i64): + v1 = load.i32 notrap aligned region0 v0 + v2 = iconst.i32 0 + store notrap aligned region0 v2, v0 + call fn0(v0) + store notrap aligned region0 v1, v0 + return +} + +; function %call_observes_cleared_flag(i64) fast { +; region0 = 0 "flags" +; sig0 = (i64) fast +; fn0 = %g sig0 +; +; block0(v0: i64): +; v1 = load.i32 notrap aligned region0 v0 +; v2 = iconst.i32 0 +; store notrap aligned region0 v2, v0 ; v2 = 0 +; call fn0(v0) +; store notrap aligned region0 v1, v0 +; return +; } + +;; Negative test: the final store writes a value other than the saved one, so it +;; is not idempotent. Only the dead middle store is removed. +function %restore_wrong_value(i64, i32) { + region0 = 0 "flags" +block0(v0: i64, v1: i32): + v2 = load.i32 notrap aligned region0 v0 + v3 = iconst.i32 0 + store notrap aligned region0 v3, v0 + store notrap aligned region0 v1, v0 + return +} + +; function %restore_wrong_value(i64, i32) fast { +; region0 = 0 "flags" +; +; block0(v0: i64, v1: i32): +; v2 = load.i32 notrap aligned region0 v0 +; store notrap aligned region0 v1, v0 +; return +; } + +;; Negative test: rolling back to the previous memory version must not resurrect +;; knowledge across a store to a *different* address in the same region. The +;; region's last-store slot is per-region, not per-address, so after the store to +;; `v0+8` the analysis no longer knows what is at `v0`, and the final store to +;; `v0` cannot be proven idempotent. +function %same_region_different_address(i64, i32) { + region0 = 0 "flags" +block0(v0: i64, v1: i32): + v2 = load.i32 notrap aligned region0 v0 + v3 = iconst.i32 0 + store notrap aligned region0 v3, v0 + store notrap aligned region0 v1, v0+8 + store notrap aligned region0 v2, v0 + return +} + +; function %same_region_different_address(i64, i32) fast { +; region0 = 0 "flags" +; +; block0(v0: i64, v1: i32): +; v2 = load.i32 notrap aligned region0 v0 +; v3 = iconst.i32 0 +; store notrap aligned region0 v3, v0 ; v3 = 0 +; store notrap aligned region0 v1, v0+8 +; store notrap aligned region0 v2, v0 +; return +; } diff --git a/cranelift/filetests/filetests/alias/endianness.clif b/cranelift/filetests/filetests/alias/endianness.clif new file mode 100644 index 000000000000..7ff7f252e645 --- /dev/null +++ b/cranelift/filetests/filetests/alias/endianness.clif @@ -0,0 +1,197 @@ +test optimize precise-output +set opt_level=speed +target x86_64 + +;; Byte order is part of a memory location's identity: two accesses to the same +;; address with the same type but opposite byte order are *not* interchangeable, +;; because forwarding a value from one to the other would drop a byte swap. + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Redundant-load elimination +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; The big-endian load must not be replaced with the little-endian load's +;; result. +function %rle_mismatched_endianness(i64) -> i16, i16 { + region0 = 0 "heap" +block0(v0: i64): + v1 = load.i16 region0 little v0 + v2 = load.i16 region0 big v0 + return v1, v2 +} + +; function %rle_mismatched_endianness(i64) -> i16, i16 fast { +; region0 = 0 "heap" +; +; block0(v0: i64): +; v1 = load.i16 little region0 v0 +; v2 = load.i16 big region0 v0 +; return v1, v2 +; } + +;; Same byte order: the second load is redundant and is removed. +function %rle_matching_endianness(i64) -> i16, i16 { + region0 = 0 "heap" +block0(v0: i64): + v1 = load.i16 region0 big v0 + v2 = load.i16 region0 big v0 + return v1, v2 +} + +; function %rle_matching_endianness(i64) -> i16, i16 fast { +; region0 = 0 "heap" +; +; block0(v0: i64): +; v1 = load.i16 big region0 v0 +; return v1, v1 +; } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Store-to-load forwarding +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; The big-endian load must not be replaced with the little-endian store's data. +function %stl_mismatched_endianness(i64, i16) -> i16 { + region0 = 0 "heap" +block0(v0: i64, v1: i16): + store.i16 region0 little v1, v0 + v2 = load.i16 region0 big v0 + return v2 +} + +; function %stl_mismatched_endianness(i64, i16) -> i16 fast { +; region0 = 0 "heap" +; +; block0(v0: i64, v1: i16): +; store little region0 v1, v0 +; v2 = load.i16 big region0 v0 +; return v2 +; } + +;; Same byte order: the load is forwarded from the store. +function %stl_matching_endianness(i64, i16) -> i16 { + region0 = 0 "heap" +block0(v0: i64, v1: i16): + store.i16 region0 big v1, v0 + v2 = load.i16 region0 big v0 + return v2 +} + +; function %stl_matching_endianness(i64, i16) -> i16 fast { +; region0 = 0 "heap" +; +; block0(v0: i64, v1: i16): +; store big region0 v1, v0 +; return v1 +; } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Idempotent-store elimination +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Storing a big-endian load's result back with little-endian byte order writes +;; the byte-swapped value to memory, so the store is not idempotent and must be +;; kept. +function %idempotent_store_mismatched_endianness(i64) { + region0 = 0 "heap" +block0(v0: i64): + v1 = load.i16 region0 big v0 + store.i16 region0 little v1, v0 + return +} + +; function %idempotent_store_mismatched_endianness(i64) fast { +; region0 = 0 "heap" +; +; block0(v0: i64): +; v1 = load.i16 big region0 v0 +; store little region0 v1, v0 +; return +; } + +;; Same byte order: the store really is idempotent and is removed. +function %idempotent_store_matching_endianness(i64) { + region0 = 0 "heap" +block0(v0: i64): + v1 = load.i16 region0 big v0 + store.i16 region0 big v1, v0 + return +} + +; function %idempotent_store_matching_endianness(i64) fast { +; region0 = 0 "heap" +; +; block0(v0: i64): +; v1 = load.i16 big region0 v0 +; return +; } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Implicit (native) versus explicit byte order +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; We do not resolve an unspecified byte order to the target's native byte +;; order, so an access that leaves the byte order implicit never forwards into +;; one that spells it out, even when they agree on this target. This is +;; conservative: it costs an optimization, but is never wrong. +function %rle_implicit_versus_explicit_endianness(i64) -> i16, i16 { + region0 = 0 "heap" +block0(v0: i64): + v1 = load.i16 region0 v0 + v2 = load.i16 region0 little v0 + return v1, v2 +} + +; function %rle_implicit_versus_explicit_endianness(i64) -> i16, i16 fast { +; region0 = 0 "heap" +; +; block0(v0: i64): +; v1 = load.i16 region0 v0 +; v2 = load.i16 little region0 v0 +; return v1, v2 +; } + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Dead-store elimination +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Dead-store elimination does not care about byte order: both stores write +;; exactly the same range of bytes, and the second one's bytes are the ones +;; that survive regardless of the order within that range. So the first store +;; is dead even though the two disagree about byte order. +function %dead_store_mismatched_endianness(i64, i16, i16) { + region0 = 0 "heap" +block0(v0: i64, v1: i16, v2: i16): + store.i16 region0 big v1, v0 + store.i16 region0 little v2, v0 + return +} + +; function %dead_store_mismatched_endianness(i64, i16, i16) fast { +; region0 = 0 "heap" +; +; block0(v0: i64, v1: i16, v2: i16): +; store little region0 v2, v0 +; return +; } + +;; Removing a dead store whose byte order differs from its overwriter's must +;; still roll the last-store state back to the version before the dead store, +;; so that the overwriter is recognized as idempotent on reprocessing. Both +;; stores should be gone here. +function %dead_store_then_idempotent_store_mismatched_endianness(i64, i16) { + region0 = 0 "heap" +block0(v0: i64, v1: i16): + v2 = load.i16 notrap region0 big v0 + store.i16 region0 little v1, v0 + store.i16 region0 big v2, v0 + return +} + +; function %dead_store_then_idempotent_store_mismatched_endianness(i64, i16) fast { +; region0 = 0 "heap" +; +; block0(v0: i64, v1: i16): +; v2 = load.i16 notrap big region0 v0 +; return +; } diff --git a/cranelift/filetests/filetests/alias/issue-14053.clif b/cranelift/filetests/filetests/alias/issue-14053.clif new file mode 100644 index 000000000000..501de5ec8628 --- /dev/null +++ b/cranelift/filetests/filetests/alias/issue-14053.clif @@ -0,0 +1,144 @@ +test optimize precise-output +set opt_level=speed +target aarch64 + +;; A *divergent* block is one that can never reach a function exit, because it +;; is part of, or only leads to, an infinite loop. But a divergent path can +;; still leave the function via an implicit trap, which makes the memory state +;; at that point observable, so dead-store elimination may not treat it as if it +;; were unreachable. + +;; The store in `block0` is overwritten by the store in `block1`, and `block1` +;; post-dominates `block0`. However, the `block2` path diverges into an infinite +;; loop which never reaches the exit, and it can trap, which must observe the +;; store from `block0`. +function %divergent_path_can_trap(i64, i32, i32, i32) { + region0 = 0 "R0" + +block0(v0: i64, v1: i32, v2: i32, v3: i32): + store notrap aligned region0 v1, v0 + brif v2, block1, block2 + +block1: + store notrap aligned region0 v2, v0 + return + +block2: + store user1 aligned region0 v3, v0+8 + jump block3 + +block3: + jump block3 +} + +; function %divergent_path_can_trap(i64, i32, i32, i32) fast { +; region0 = 0 "R0" +; +; block0(v0: i64, v1: i32, v2: i32, v3: i32): +; store notrap aligned region0 v1, v0 +; brif v2, block1, block2 +; +; block1: +; store.i32 notrap aligned region0 v2, v0 +; return +; +; block2: +; store.i32 user1 aligned region0 v3, v0+8 +; jump block3 +; +; block3: +; jump block3 +; } + +;; Same as above, except that the divergent path stores to `region0` before it +;; can trap. +function %shadowed_before_trap_on_divergent_path(i64, i32, i32, i32) { + region0 = 0 "R0" + +block0(v0: i64, v1: i32, v2: i32, v3: i32): + store notrap aligned region0 v1, v0 + brif v2, block1, block2 + +block1: + store notrap aligned region0 v2, v0 + return + +block2: + store notrap aligned region0 v3, v0+8 + store user1 aligned region0 v3, v0+16 + jump block3 + +block3: + jump block3 +} + +; function %shadowed_before_trap_on_divergent_path(i64, i32, i32, i32) fast { +; region0 = 0 "R0" +; +; block0(v0: i64, v1: i32, v2: i32, v3: i32): +; store notrap aligned region0 v1, v0 +; brif v2, block1, block2 +; +; block1: +; store.i32 notrap aligned region0 v2, v0 +; return +; +; block2: +; store.i32 notrap aligned region0 v3, v0+8 +; store.i32 user1 aligned region0 v3, v0+16 +; jump block3 +; +; block3: +; jump block3 +; } + +;; Both stores are themselves in divergent blocks. +function %both_stores_in_divergent_blocks(i64, i32, i32, i32) { + region0 = 0 "R0" + +block0(v0: i64, v1: i32, v2: i32, v3: i32): + brif v2, block1, block5 + +block1: + store notrap aligned region0 v1, v0 + brif v3, block2, block3 + +block2: + store user1 aligned region0 v3, v0+8 + jump block4 + +block3: + store notrap aligned region0 v2, v0 + jump block4 + +block4: + jump block4 + +block5: + return +} + +; function %both_stores_in_divergent_blocks(i64, i32, i32, i32) fast { +; region0 = 0 "R0" +; +; block0(v0: i64, v1: i32, v2: i32, v3: i32): +; brif v2, block1, block5 +; +; block1: +; store.i32 notrap aligned region0 v1, v0 +; brif.i32 v3, block2, block3 +; +; block2: +; store.i32 user1 aligned region0 v3, v0+8 +; jump block4 +; +; block3: +; store.i32 notrap aligned region0 v2, v0 +; jump block4 +; +; block4: +; jump block4 +; +; block5: +; return +; } diff --git a/cranelift/filetests/filetests/alias/issue-14131-atomic.clif b/cranelift/filetests/filetests/alias/issue-14131-atomic.clif new file mode 100644 index 000000000000..f0a68a79461b --- /dev/null +++ b/cranelift/filetests/filetests/alias/issue-14131-atomic.clif @@ -0,0 +1,40 @@ +test optimize precise-output +set opt_level=speed_and_size +target x86_64 + +;; Regression test for https://github.com/bytecodealliance/wasmtime/issues/14131 +;; +;; The last-store slot that dead-store elimination must not materialize into +;; `region0` holds an *atomic* store rather than a regionless store. +;; +;; The `atomic_store` has memory fence semantics, so it becomes the `last_fence` +;; while `region0`'s last-store slot stays empty. The first `region0` store +;; therefore sees the `last_fence` fallback, and is then made dead by the second +;; `region0` store. Unwinding the last-store state past the removed store must +;; restore `region0`'s empty slot rather than the last-fence fallback: otherwise +;; reprocessing the overwriting store "observes" the atomic store, an +;; observation the up-front observed-stores analysis never makes (the trailing +;; `fence` replaces the atomic in `last_fence` without observing it), ultimately +;; leading to an assertion failure. + +function %f(i64, i32) system_v { + region0 = 1 "table" + +block0(v0: i64, v1: i32): + atomic_store.i32 notrap v1, v0 + store notrap region0 v1, v0 + store notrap region0 v1, v0 + fence + return +} + +; function %f(i64, i32) system_v { +; region0 = 1 "table" +; +; block0(v0: i64, v1: i32): +; atomic_store notrap v1, v0 +; store notrap region0 v1, v0 +; fence +; return +; } + diff --git a/cranelift/filetests/filetests/alias/issue-14131.clif b/cranelift/filetests/filetests/alias/issue-14131.clif new file mode 100644 index 000000000000..e16a0833b6e2 --- /dev/null +++ b/cranelift/filetests/filetests/alias/issue-14131.clif @@ -0,0 +1,38 @@ +test optimize precise-output +set opt_level=speed_and_size +target x86_64 + +;; Regression test for https://github.com/bytecodealliance/wasmtime/issues/14131 +;; +;; The first store has no alias region, so it is treated as a fence and becomes +;; the `last_fence`, while `region0`'s last-store slot stays empty. The second +;; store is then a dead store, overwritten by the third. When dead-store +;; elimination unwinds the last-store state past the store it just removed, it +;; must not materialize the `last_fence` fallback into `region0`'s slot: +;; reprocessing the overwriting store would then "observe" the regionless store, +;; an observation that the up-front observed-stores analysis never made, which +;; leads to an assertion failure. + +function %f(i64, i64) system_v { + region0 = 1 "table" + +block0(v0: i64, v1: i64): + v2 = iconst.i32 0 + store notrap v2, v1 + store notrap region0 v2, v0 + store notrap region0 v2, v0 + fence + return +} + +; function %f(i64, i64) system_v { +; region0 = 1 "table" +; +; block0(v0: i64, v1: i64): +; v2 = iconst.i32 0 +; store notrap v2, v1 ; v2 = 0 +; store notrap region0 v2, v0 ; v2 = 0 +; fence +; return +; } + diff --git a/cranelift/filetests/filetests/runtests/alias-analysis-endianness.clif b/cranelift/filetests/filetests/runtests/alias-analysis-endianness.clif new file mode 100644 index 000000000000..bb53ed0c797e --- /dev/null +++ b/cranelift/filetests/filetests/runtests/alias-analysis-endianness.clif @@ -0,0 +1,51 @@ +test interpret +test run +set opt_level=speed + +;; XXX: only the s390x and Pulley backends implement non-native byte order for +;; memory accesses at the time of writing, and these tests need a mismatched +;; pair to exercise the alias-analysis behavior at all. +target s390x +target pulley64 + +;; Redundant-load elimination between a little-endian and a big-endian load: +;; the big-endian load must not be replaced with the little-endian one's +;; result. Correct result: 0x1234 ^ 0x3412 == 0x2626. +function %rle_across_endianness(i16) -> i16 { + ss0 = explicit_slot 8 +block0(v1: i16): + v0 = stack_addr.i64 ss0 + store little v1, v0 + v2 = load.i16 little v0 + v3 = load.i16 big v0 + v4 = bxor v2, v3 + return v4 +} +; run: %rle_across_endianness(0x1234) == 0x2626 + +;; Store-to-load forwarding between a little-endian store and a big-endian +;; load: the load must not be replaced with the store's data. +function %stl_across_endianness(i16) -> i16 { + ss0 = explicit_slot 8 +block0(v1: i16): + v0 = stack_addr.i64 ss0 + store little v1, v0 + v2 = load.i16 big v0 + return v2 +} +; run: %stl_across_endianness(0x1234) == 0x3412 + +;; Idempotent-store elimination between a big-endian load and a little-endian +;; store: the store writes the byte-swapped value, so it is not idempotent and +;; must not be deleted. +function %idempotent_store_across_endianness(i16) -> i16 { + ss0 = explicit_slot 8 +block0(v1: i16): + v0 = stack_addr.i64 ss0 + store little v1, v0 + v2 = load.i16 big v0 + store little v2, v0 + v3 = load.i16 little v0 + return v3 +} +; run: %idempotent_store_across_endianness(0x1234) == 0x3412 diff --git a/cranelift/frontend/Cargo.toml b/cranelift/frontend/Cargo.toml index bd075d0cf229..f3f09090a383 100644 --- a/cranelift/frontend/Cargo.toml +++ b/cranelift/frontend/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-frontend" -version = "0.135.0" +version = "0.135.1" description = "Cranelift IR builder helper" license = "Apache-2.0 WITH LLVM-exception" documentation = "https://docs.rs/cranelift-frontend" diff --git a/cranelift/interpreter/Cargo.toml b/cranelift/interpreter/Cargo.toml index d4e6dc02828e..5cbfffef93ea 100644 --- a/cranelift/interpreter/Cargo.toml +++ b/cranelift/interpreter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cranelift-interpreter" -version = "0.135.0" +version = "0.135.1" authors = ["The Cranelift Project Developers"] description = "Interpret Cranelift IR" repository = "https://github.com/bytecodealliance/wasmtime" diff --git a/cranelift/isle/isle/Cargo.toml b/cranelift/isle/isle/Cargo.toml index bb1fcd1aaa4c..04cc98aaaddb 100644 --- a/cranelift/isle/isle/Cargo.toml +++ b/cranelift/isle/isle/Cargo.toml @@ -7,7 +7,7 @@ license = "Apache-2.0 WITH LLVM-exception" name = "cranelift-isle" readme = "../README.md" repository = "https://github.com/bytecodealliance/wasmtime/tree/main/cranelift/isle" -version = "0.135.0" +version = "0.135.1" [lints] workspace = true diff --git a/cranelift/jit/Cargo.toml b/cranelift/jit/Cargo.toml index 50b95dda5321..fe45317a0ef2 100644 --- a/cranelift/jit/Cargo.toml +++ b/cranelift/jit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cranelift-jit" -version = "0.135.0" +version = "0.135.1" authors = ["The Cranelift Project Developers"] description = "A JIT library backed by Cranelift" repository = "https://github.com/bytecodealliance/wasmtime" diff --git a/cranelift/module/Cargo.toml b/cranelift/module/Cargo.toml index c2638062d381..1aed12eb97ab 100644 --- a/cranelift/module/Cargo.toml +++ b/cranelift/module/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cranelift-module" -version = "0.135.0" +version = "0.135.1" authors = ["The Cranelift Project Developers"] description = "Support for linking functions and data with Cranelift" repository = "https://github.com/bytecodealliance/wasmtime" diff --git a/cranelift/native/Cargo.toml b/cranelift/native/Cargo.toml index e592b3005b3c..c98e90bdc7e6 100644 --- a/cranelift/native/Cargo.toml +++ b/cranelift/native/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cranelift-native" -version = "0.135.0" +version = "0.135.1" authors = ["The Cranelift Project Developers"] description = "Support for targeting the host with Cranelift" documentation = "https://docs.rs/cranelift-native" diff --git a/cranelift/object/Cargo.toml b/cranelift/object/Cargo.toml index 32cfe6712d64..4244864940e9 100644 --- a/cranelift/object/Cargo.toml +++ b/cranelift/object/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cranelift-object" -version = "0.135.0" +version = "0.135.1" authors = ["The Cranelift Project Developers"] description = "Emit Cranelift output to native object files with `object`" repository = "https://github.com/bytecodealliance/wasmtime" diff --git a/cranelift/reader/Cargo.toml b/cranelift/reader/Cargo.toml index 278a3905c99e..db800f557031 100644 --- a/cranelift/reader/Cargo.toml +++ b/cranelift/reader/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift-reader" -version = "0.135.0" +version = "0.135.1" description = "Cranelift textual IR reader" license = "Apache-2.0 WITH LLVM-exception" documentation = "https://docs.rs/cranelift-reader" diff --git a/cranelift/serde/Cargo.toml b/cranelift/serde/Cargo.toml index 40cc191701c3..2201b3eacc85 100644 --- a/cranelift/serde/Cargo.toml +++ b/cranelift/serde/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cranelift-serde" -version = "0.135.0" +version = "0.135.1" authors = ["The Cranelift Project Developers"] description = "Serializer/Deserializer for Cranelift IR" repository = "https://github.com/bytecodealliance/wasmtime" diff --git a/cranelift/srcgen/Cargo.toml b/cranelift/srcgen/Cargo.toml index 424a6e50c391..24e92f3c614d 100644 --- a/cranelift/srcgen/Cargo.toml +++ b/cranelift/srcgen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cranelift-srcgen" -version = "0.135.0" +version = "0.135.1" authors = ["The Wasmtime Project Developers"] description = "Helper functions for generating Rust and ISLE files" license = "Apache-2.0 WITH LLVM-exception" diff --git a/cranelift/umbrella/Cargo.toml b/cranelift/umbrella/Cargo.toml index e96ca75d8bff..62f56ee5cf54 100644 --- a/cranelift/umbrella/Cargo.toml +++ b/cranelift/umbrella/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["The Cranelift Project Developers"] name = "cranelift" -version = "0.135.0" +version = "0.135.1" description = "Umbrella for commonly-used cranelift crates" license = "Apache-2.0 WITH LLVM-exception" documentation = "https://docs.rs/cranelift" diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index 00ac5888d4fd..29a7ec70b233 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -229,7 +229,7 @@ /** * \brief Wasmtime version string. */ -#define WASMTIME_VERSION "48.0.0" +#define WASMTIME_VERSION "48.0.1" /** * \brief Wasmtime major version number. */ @@ -241,6 +241,6 @@ /** * \brief Wasmtime patch version number. */ -#define WASMTIME_VERSION_PATCH 0 +#define WASMTIME_VERSION_PATCH 1 #endif // WASMTIME_API_H diff --git a/crates/c-api/src/async.rs b/crates/c-api/src/async.rs index a3d079001949..6e7b68bbd7ee 100644 --- a/crates/c-api/src/async.rs +++ b/crates/c-api/src/async.rs @@ -288,9 +288,9 @@ pub unsafe extern "C" fn wasmtime_linker_define_async_func( finalizer: Option, ) -> Option> { let ty = ty.ty().ty(linker.linker.engine()); + let cb = c_async_callback_to_rust_fn(callback, data, finalizer); let module = to_str!(module, module_len); let name = to_str!(name, name_len); - let cb = c_async_callback_to_rust_fn(callback, data, finalizer); handle_result( linker.linker.func_new_async(module, name, ty, cb), diff --git a/crates/c-api/src/component/linker.rs b/crates/c-api/src/component/linker.rs index 3823b8b69324..e4714f769cf8 100644 --- a/crates/c-api/src/component/linker.rs +++ b/crates/c-api/src/component/linker.rs @@ -115,13 +115,13 @@ pub unsafe extern "C" fn wasmtime_component_linker_instance_add_func( data: *mut c_void, finalizer: Option, ) -> Option> { + let foreign = crate::ForeignData { data, finalizer }; + let name = unsafe { std::slice::from_raw_parts(name, name_len) }; let Ok(name) = std::str::from_utf8(name) else { return crate::bad_utf8(); }; - let foreign = crate::ForeignData { data, finalizer }; - let result = linker_instance .linker_instance .func_new(&name, move |ctx, ty, args, rets| { @@ -181,13 +181,13 @@ pub unsafe extern "C" fn wasmtime_component_linker_instance_add_func_async( data: *mut c_void, finalizer: Option, ) -> Option> { + let foreign = crate::ForeignData { data, finalizer }; + let name = unsafe { std::slice::from_raw_parts(name, name_len) }; let Ok(name) = std::str::from_utf8(name) else { return crate::bad_utf8(); }; - let foreign = crate::ForeignData { data, finalizer }; - let result = linker_instance .linker_instance @@ -322,13 +322,13 @@ pub unsafe extern "C" fn wasmtime_component_linker_instance_add_resource( data: *mut c_void, finalizer: Option, ) -> Option> { + let foreign = crate::ForeignData { data, finalizer }; + let name = unsafe { std::slice::from_raw_parts(name, name_len) }; let Ok(name) = std::str::from_utf8(name) else { return crate::bad_utf8(); }; - let foreign = crate::ForeignData { data, finalizer }; - let result = linker_instance .linker_instance .resource(name, ty.ty, move |ctx, rep| { diff --git a/crates/c-api/src/linker.rs b/crates/c-api/src/linker.rs index f7b1da6308bc..b21422a8bc56 100644 --- a/crates/c-api/src/linker.rs +++ b/crates/c-api/src/linker.rs @@ -79,9 +79,9 @@ pub unsafe extern "C" fn wasmtime_linker_define_func( finalizer: Option, ) -> Option> { let ty = ty.ty().ty(linker.linker.engine()); + let cb = crate::func::c_callback_to_rust_fn(callback, data, finalizer); let module = to_str!(module, module_len); let name = to_str!(name, name_len); - let cb = crate::func::c_callback_to_rust_fn(callback, data, finalizer); handle_result(linker.linker.func_new(module, name, ty, cb), |_linker| ()) } @@ -98,9 +98,9 @@ pub unsafe extern "C" fn wasmtime_linker_define_func_unchecked( finalizer: Option, ) -> Option> { let ty = ty.ty().ty(linker.linker.engine()); + let cb = crate::func::c_unchecked_callback_to_rust_fn(callback, data, finalizer); let module = to_str!(module, module_len); let name = to_str!(name, name_len); - let cb = crate::func::c_unchecked_callback_to_rust_fn(callback, data, finalizer); handle_result( linker.linker.func_new_unchecked(module, name, ty, cb), |_linker| (), diff --git a/crates/c-api/tests/async.cc b/crates/c-api/tests/async.cc index 7f396187835e..7785ed1664ac 100644 --- a/crates/c-api/tests/async.cc +++ b/crates/c-api/tests/async.cc @@ -6,6 +6,33 @@ using namespace wasmtime; +namespace { + +void async_callback(void *, wasmtime_caller_t *, const wasmtime_val_t *, size_t, + wasmtime_val_t *, size_t, wasm_trap_t **, + wasmtime_async_continuation_t *) {} + +void finalize(void *data) { *static_cast(data) = true; } + +} // namespace + +TEST(async, finalizes_callback_when_name_parsing_fails) { + Engine engine; + Linker linker(engine); + auto *ty = wasm_functype_new_0_0(); + const char invalid_utf8[] = {static_cast(0xff)}; + bool finalized = false; + + auto *error = wasmtime_linker_define_async_func( + linker.capi(), invalid_utf8, sizeof(invalid_utf8), "name", 4, ty, + async_callback, &finalized, finalize); + + ASSERT_NE(error, nullptr); + wasmtime_error_delete(error); + EXPECT_TRUE(finalized); + wasm_functype_delete(ty); +} + TEST(async, call_func_async) { Engine engine; Store store(engine); diff --git a/crates/c-api/tests/component/linker.cc b/crates/c-api/tests/component/linker.cc index 88158a3c4763..c928aa799951 100644 --- a/crates/c-api/tests/component/linker.cc +++ b/crates/c-api/tests/component/linker.cc @@ -4,6 +4,63 @@ using namespace wasmtime::component; +static wasmtime_error_t *func_callback(void *, wasmtime_context_t *, + const wasmtime_component_func_type_t *, + wasmtime_component_val_t *, size_t, + wasmtime_component_val_t *, size_t) { + return nullptr; +} + +static void async_func_callback(void *, wasmtime_context_t *, + const wasmtime_component_func_type_t *, + wasmtime_component_val_t *, size_t, + wasmtime_component_val_t *, size_t, + wasmtime_error_t **, + wasmtime_async_continuation_t *) {} + +static wasmtime_error_t *resource_destructor(void *, wasmtime_context_t *, + uint32_t) { + return nullptr; +} + +static void finalize(void *data) { *static_cast(data) = true; } + +TEST(Linker, finalizes_callbacks_when_name_parsing_fails) { + wasmtime::Engine engine; + auto *raw = wasmtime_component_linker_new(engine.capi()); + auto *root = wasmtime_component_linker_root(raw); + const char invalid_utf8[] = {static_cast(0xff)}; + + bool finalized = false; + auto *error = wasmtime_component_linker_instance_add_func( + root, invalid_utf8, sizeof(invalid_utf8), func_callback, &finalized, + finalize); + ASSERT_NE(error, nullptr); + wasmtime_error_delete(error); + EXPECT_TRUE(finalized); + + finalized = false; + error = wasmtime_component_linker_instance_add_func_async( + root, invalid_utf8, sizeof(invalid_utf8), async_func_callback, &finalized, + finalize); + ASSERT_NE(error, nullptr); + wasmtime_error_delete(error); + EXPECT_TRUE(finalized); + + finalized = false; + auto *ty = wasmtime_component_resource_type_new_host(0); + error = wasmtime_component_linker_instance_add_resource( + root, invalid_utf8, sizeof(invalid_utf8), ty, resource_destructor, + &finalized, finalize); + ASSERT_NE(error, nullptr); + wasmtime_error_delete(error); + EXPECT_TRUE(finalized); + wasmtime_component_resource_type_delete(ty); + + wasmtime_component_linker_instance_delete(root); + wasmtime_component_linker_delete(raw); +} + TEST(Linker, allow_shadowing) { wasmtime::Engine engine; Linker linker(engine); diff --git a/crates/c-api/tests/linker.cc b/crates/c-api/tests/linker.cc index 3f99af4f4d85..730c6b3a3ba7 100644 --- a/crates/c-api/tests/linker.cc +++ b/crates/c-api/tests/linker.cc @@ -1,9 +1,26 @@ #include +#include #include #include using namespace wasmtime; +namespace { + +wasm_trap_t *callback(void *, wasmtime_caller_t *, const wasmtime_val_t *, + size_t, wasmtime_val_t *, size_t) { + return nullptr; +} + +wasm_trap_t *unchecked_callback(void *, wasmtime_caller_t *, + wasmtime_val_raw_t *, size_t) { + return nullptr; +} + +void finalize(void *data) { *static_cast(data) = true; } + +} // namespace + TEST(Linker, Smoke) { Engine engine; Linker linker(engine); @@ -75,6 +92,31 @@ TEST(Linker, CallableCopy) { linker.func_new("a", "f", FuncType({}, {}), cf).unwrap(); } +TEST(Linker, FinalizesCallbacksWhenNameParsingFails) { + Engine engine; + Linker linker(engine); + auto *ty = wasm_functype_new_0_0(); + const char invalid_utf8[] = {static_cast(0xff)}; + + bool finalized = false; + auto *error = wasmtime_linker_define_func(linker.capi(), invalid_utf8, + sizeof(invalid_utf8), "name", 4, ty, + callback, &finalized, finalize); + ASSERT_NE(error, nullptr); + wasmtime_error_delete(error); + EXPECT_TRUE(finalized); + + finalized = false; + error = wasmtime_linker_define_func_unchecked( + linker.capi(), "module", 6, invalid_utf8, sizeof(invalid_utf8), ty, + unchecked_callback, &finalized, finalize); + ASSERT_NE(error, nullptr); + wasmtime_error_delete(error); + EXPECT_TRUE(finalized); + + wasm_functype_delete(ty); +} + TEST(Linker, DefineUnknownImportsAsTraps) { Engine engine; Linker linker(engine); diff --git a/crates/component-macro/tests/codegen.rs b/crates/component-macro/tests/codegen.rs index f40ea9136ee1..dd237ba79b87 100644 --- a/crates/component-macro/tests/codegen.rs +++ b/crates/component-macro/tests/codegen.rs @@ -861,6 +861,31 @@ mod named_imports { } } + mod hyphenated_interface_name { + wasmtime::component::bindgen!({ + inline: " + package foo:foo; + + interface my-itf { + ping: func(); + } + + world the-world { + import my-itf; + } + ", + named_imports: { + "foo:foo/my-itf": String, + }, + }); + + struct MyHost; + + impl named_imports::foo::foo::my_itf::Host for MyHost { + fn ping(&mut self, _id: String) {} + } + } + mod async_store { #[derive(Clone)] pub struct MyId(u32); diff --git a/crates/cranelift/src/debug/transform/simulate.rs b/crates/cranelift/src/debug/transform/simulate.rs index dd39c2f54398..2504ea32653a 100644 --- a/crates/cranelift/src/debug/transform/simulate.rs +++ b/crates/cranelift/src/debug/transform/simulate.rs @@ -296,8 +296,15 @@ pub fn generate_simulated_dwarf( out_strings: &mut write::StringTable, isa: &dyn TargetIsa, ) -> Result<(), Error> { + // A component without any core modules has no functions to describe, and + // the compilation unit below names itself after the first translation's + // wasm file. There is nothing to simulate, so leave the DWARF empty. + let Some((_, first_translation)) = compilation.translations.iter().next() else { + return Ok(()); + }; + let (wasm_file, path) = { - let di = &compilation.translations.iter().next().unwrap().1.debuginfo; + let di = &first_translation.debuginfo; let path = di .wasm_file .path diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 10d20e63b170..64e9642bf584 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -663,85 +663,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builder.switch_to_block(continuation_block); } - /// Manually insert a fuel check, as opposed to what already happens around - /// normal loops headers and function entries. - /// - /// This can be used for expensive opcodes, such as `array.copy`, where the - /// operation's runtime is a function of the runtime state. - fn manual_fuel_check(&mut self, builder: &mut FunctionBuilder<'_>, fuel_to_consume: ir::Value) { - self.fuel_increment_var(builder); - - let fuel = builder.use_var(self.fuel_var); - let fuel = builder.ins().iadd(fuel, fuel_to_consume); - builder.def_var(self.fuel_var, fuel); - - self.fuel_check(builder); - } - - /// Consumes `units * cost_per_unit` fuel, saturating the charge at - /// `i64::MAX` so that an oversized unsigned operand cannot wrap around and - /// add fuel instead. - fn consume_variable_fuel( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - ) { - let may_exceed_i64_max = match builder.func.dfg.value_type(units) { - I32 => false, - I64 => true, - ty => unreachable!("unsupported variable fuel unit type: {ty}"), - }; - self.consume_variable_fuel_impl(builder, units, cost_per_unit, may_exceed_i64_max); - } - - /// Like [`Self::consume_variable_fuel`], but for a unit count whose product - /// with `cost_per_unit` is statically known to fit in an `i64`. - fn consume_bounded_variable_fuel( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - ) { - self.consume_variable_fuel_impl(builder, units, cost_per_unit, false); - } - - fn consume_variable_fuel_impl( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - may_exceed_i64_max: bool, - ) { - if !self.tunables.consume_fuel || cost_per_unit == 0 { - return; - } - - let units = match builder.func.dfg.value_type(units) { - I32 => builder.ins().uextend(I64, units), - I64 => units, - ty => unreachable!("unsupported variable fuel unit type: {ty}"), - }; - let fuel = if cost_per_unit == 1 { - units - } else { - builder.ins().imul_imm_s(units, i64::from(cost_per_unit)) - }; - let fuel = if may_exceed_i64_max { - let max = builder.ins().iconst(I64, i64::MAX); - let max_units = builder - .ins() - .iconst(I64, i64::MAX / i64::from(cost_per_unit)); - let saturate = builder - .ins() - .icmp(IntCC::UnsignedGreaterThan, units, max_units); - builder.ins().select(saturate, max, fuel) - } else { - fuel - }; - self.manual_fuel_check(builder, fuel); - } - fn epoch_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) { debug_assert!(self.epoch_deadline_var.is_reserved_value()); self.epoch_deadline_var = builder.declare_var(ir::types::I64); @@ -2684,17 +2605,19 @@ impl FuncEnvironment<'_> { delta: ir::Value, init_value: ir::Value, ) -> WasmResult { + let cost = self + .tunables + .operator_cost + .variable() + .table_grow_per_element; + self.pre_translate_bulk_op(builder, delta, cost)?; + let mut pos = builder.cursor(); let table = self.table(table_index); let (table_vmctx, defined_table_index) = self.table_vmctx_and_defined_index(&mut pos, table_index); let index_type = table.idx_type; let delta64 = self.cast_index_to_i64(&mut pos, delta, index_type); - let cost = self - .tunables - .operator_cost - .variable() - .table_grow_per_element; // Call out to the host to perform the actual growth of the underlying // table. This will initialize table slots as all null. Afterwards the @@ -2740,7 +2663,6 @@ impl FuncEnvironment<'_> { // A failed attempt performs no initialization loop, but still charge // for the requested growth so repeated failures are not free. builder.switch_to_block(failed_block); - self.consume_variable_fuel(builder, delta, cost); builder.ins().jump(done_block, &[]); builder.switch_to_block(fill_block); @@ -2748,12 +2670,11 @@ impl FuncEnvironment<'_> { builder, CheckedEntity::Table { table: table_index, - initialized: true, + initialized: false, }, result_idx, init_value, delta, - cost, )?; builder.ins().jump(done_block, &[]); @@ -2907,6 +2828,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -2916,7 +2838,6 @@ impl FuncEnvironment<'_> { dst, val, len, - cost, ) } @@ -3053,8 +2974,7 @@ impl FuncEnvironment<'_> { elem: ir::Value, len: ir::Value, ) -> WasmResult { - let cost = self.tunables.operator_cost.variable().array_new_per_element; - gc::translate_array_new(self, builder, array_type_index, elem, len, cost) + gc::translate_array_new(self, builder, array_type_index, elem, len) } pub fn translate_array_new_default( @@ -3063,12 +2983,7 @@ impl FuncEnvironment<'_> { array_type_index: TypeIndex, len: ir::Value, ) -> WasmResult { - let cost = self - .tunables - .operator_cost - .variable() - .array_new_default_per_element; - gc::translate_array_new_default(self, builder, array_type_index, len, cost) + gc::translate_array_new_default(self, builder, array_type_index, len) } pub fn translate_array_new_fixed( @@ -3151,6 +3066,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_copy_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3166,7 +3082,6 @@ impl FuncEnvironment<'_> { dst_index, src_index, len, - cost, ) } @@ -3185,6 +3100,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Array { @@ -3195,7 +3111,6 @@ impl FuncEnvironment<'_> { index, value, len, - cost, ) } @@ -3216,6 +3131,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_init_data_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3230,7 +3146,6 @@ impl FuncEnvironment<'_> { dst_index, data_offset, len, - cost, ) } @@ -3250,6 +3165,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_init_elem_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3261,7 +3177,6 @@ impl FuncEnvironment<'_> { dst_index, elem_offset, len, - cost, ) } @@ -3634,7 +3549,7 @@ impl FuncEnvironment<'_> { let index_type = self.memory(index).idx_type; let cost = self.tunables.operator_cost.variable().memory_grow_per_page; - self.consume_variable_fuel(builder, val, cost); + self.pre_translate_bulk_op(builder, val, cost)?; let mut pos = builder.cursor(); let val = self.cast_index_to_i64(&mut pos, val, index_type); let call_inst = pos @@ -3733,7 +3648,8 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let cost = self.tunables.operator_cost.variable().memory_copy_per_byte; - self.translate_entity_copy(builder, dst_index, src_index, dst, src, len, cost) + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_copy(builder, dst_index, src_index, dst, src, len) } /// Perform a raw bulk-memory-like libcall. @@ -3741,7 +3657,11 @@ impl FuncEnvironment<'_> { /// The main purpose of this helper is to handle situations when fuel and /// epochs are enabled to break up the copy into a loop of chunks with /// preemption checks between them. - fn raw_bulk_memory_operation(&mut self, builder: &mut FunctionBuilder<'_>, mut op: BulkOp) { + fn raw_bulk_memory_operation( + &mut self, + builder: &mut FunctionBuilder<'_>, + op: BulkOp, + ) -> WasmResult<()> { // Fast path: a copy whose byte length is a small compile-time constant is // expanded inline (see `emit_inline_memcpy`), skipping the libcall's fixed // per-call cost (a wasm/host transition and an indirect call) that @@ -3758,259 +3678,33 @@ impl FuncEnvironment<'_> { const_len: Some(bytes), src_entity, dst_entity, - fuel, .. } = op { if bytes <= INLINE_COPY_MAX_BYTES { - if self.tunables.consume_fuel { - let units = bytes / u64::from(fuel.bytes_per_unit); - self.fuel_consumed += - i64::try_from(units * u64::from(fuel.cost_per_unit)).unwrap(); - } let src_region = self.bulk_copy_alias_region(builder.func, src_entity); let dst_region = self.bulk_copy_alias_region(builder.func, dst_entity); self.emit_inline_memcpy(builder, dst, src, bytes, src_region, dst_region); - return; + return Ok(()); } } - // Very scientifically chosen. Or, more seriously, this is just an - // arbitrary number for now. 100k copies of this size locally takes half - // a second, so seems like a reasonably large chunk size to not hit perf - // too much by chunking but also enable time slicing. - const UNINTERRUPTABLE_CHUNK_SIZE: i64 = 128 << 20; - let mut pos = builder.cursor(); let vmctx = self.vmctx_val(&mut pos); - let pointer_type = self.pointer_type(); // Performs a raw call to the actual libcall, as dictated by the - // provided `op`. This inserts configured epoch/fuel checks before the - // call. - let raw_call = - |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &BulkOp| { - if env.tunables.epoch_interruption { - env.epoch_check(builder); - } - let fuel = op.fuel(); - if env.tunables.consume_fuel && fuel.cost_per_unit != 0 { - let byte_len = op.len(); - debug_assert!(fuel.bytes_per_unit.is_power_of_two()); - let units = if fuel.bytes_per_unit == 1 { - byte_len - } else { - builder - .ins() - .ushr_imm_u(byte_len, i64::from(fuel.bytes_per_unit.trailing_zeros())) - }; - // With fuel enabled all calls emitted below are limited to - // `UNINTERRUPTABLE_CHUNK_SIZE`, so this multiplication - // cannot exceed `i64::MAX` even at the maximum `u8` rate. - env.consume_bounded_variable_fuel(builder, units, fuel.cost_per_unit); - } - match *op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - let memory_copy = env.builtin_functions.memory_copy(&mut builder.func); - builder.ins().call(memory_copy, &[vmctx, dst, src, len]); - } - BulkOp::MemoryFill { dst, val, len, .. } => { - let memory_fill = env.builtin_functions.memory_fill(&mut builder.func); - builder.ins().call(memory_fill, &[vmctx, dst, val, len]); - } - } - }; - - // If epochs and fuel are disabled, then just call the libcall and - // return. No need for the loops below. - if !self.tunables.epoch_interruption && !self.tunables.consume_fuel { - raw_call(self, builder, &op); - return; - } - - // If fuel is enabled, first take all the pending fuel and flush it to - // our internal variable. This is necessary to avoid picking up all - // pending fuel on each turn of the loop below. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - - let current_block = builder.current_block().unwrap(); - let chunk_block = builder.create_block(); - let last_chunk_block = builder.create_block(); - - builder.ensure_inserted_block(); - builder.insert_block_after(chunk_block, current_block); - builder.insert_block_after(last_chunk_block, chunk_block); - - let chunk = builder - .ins() - .iconst(pointer_type, UNINTERRUPTABLE_CHUNK_SIZE); - - // For `memcpy` when chunking this up we might need to do a backwards - // copy or a forwards copy. Determine that here and jump to the - // backwards copy if needed. - let backwards_block = if let BulkOp::MemoryCopy { dst, src, .. } = op { - let forwards = builder.ins().icmp(IntCC::UnsignedGreaterThan, src, dst); - let forwards_block = builder.create_block(); - let backwards_block = builder.create_block(); - builder - .ins() - .brif(forwards, forwards_block, &[], backwards_block, &[]); - builder.switch_to_block(forwards_block); - builder.seal_block(forwards_block); - Some((backwards_block, op.clone())) - } else { - None - }; - - // Helper closure to test if the length in `op` is larger than `chunk`, - // and if so do a single chunk. Else this goes to the final block with - // the final operation. - let has_chunk_branch = - |builder: &mut FunctionBuilder<'_>, op: &_, chunk_block, last_chunk_block| { - let len = match *op { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len, - }; - let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk); - match *op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - builder.ins().brif( - has_chunk, - chunk_block, - &[dst.into(), src.into(), len.into()], - last_chunk_block, - &[dst.into(), src.into(), len.into()], - ); - } - BulkOp::MemoryFill { dst, len, .. } => { - builder.ins().brif( - has_chunk, - chunk_block, - &[dst.into(), len.into()], - last_chunk_block, - &[dst.into(), len.into()], - ); - } - } - }; - - let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - *dst = builder.append_block_param(block, pointer_type); - *src = builder.append_block_param(block, pointer_type); - *len = builder.append_block_param(block, pointer_type); - } - BulkOp::MemoryFill { dst, len, .. } => { - *dst = builder.append_block_param(block, pointer_type); - *len = builder.append_block_param(block, pointer_type); - } - }; - - // Forwards copy: dispatch to the per-chunk loop or the final iteration - // if there's no chunks. - has_chunk_branch(builder, &op, chunk_block, last_chunk_block); - - // Forwards copy: In the block with per-chunk copies, each operation - // performs `chunk` length of bytes and then decrements the current - // length by `chunk`. Afterwards a condition tests if we do another - // chunk or break out for the final chunk. - builder.switch_to_block(chunk_block); - append_block_params(builder, chunk_block, &mut op); - let op_len = match &mut op { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len, - }; - let remaining_len = *op_len; - *op_len = chunk; - raw_call(self, builder, &op); - match &mut op { + // provided `op`. + match op { BulkOp::MemoryCopy { dst, src, len, .. } => { - *dst = builder.ins().iadd(*dst, chunk); - *src = builder.ins().iadd(*src, chunk); - *len = builder.ins().isub(remaining_len, chunk); + let memory_copy = self.builtin_functions.memory_copy(&mut builder.func); + builder.ins().call(memory_copy, &[vmctx, dst, src, len]); } - BulkOp::MemoryFill { len, dst, .. } => { - *dst = builder.ins().iadd(*dst, chunk); - *len = builder.ins().isub(remaining_len, chunk); + BulkOp::MemoryFill { dst, val, len } => { + let memory_fill = self.builtin_functions.memory_fill(&mut builder.func); + builder.ins().call(memory_fill, &[vmctx, dst, val, len]); } - }; - has_chunk_branch(builder, &op, chunk_block, last_chunk_block); - builder.seal_block(chunk_block); - - // Backwards copy: similar to the above but with adjustments on where - // increments/decrements happen. Notably: - // - // * Initial src/end are the final byte address - // * Each chunk starts out by decrementing src/end as opposed to above - // where the increment happens at the end. - // * The final block performs the final decrement before jumping to the - // shared `last_chunk_block` between the forwards/backwards paths. - if let Some((backwards_block, mut op)) = backwards_block { - // Setup `dst=dst+len` and `src=src+len`, then see if we have a - // chunk. - builder.switch_to_block(backwards_block); - builder.seal_block(backwards_block); - let backwards_chunk_block = builder.create_block(); - let backwards_last_chunk_block = builder.create_block(); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - *dst = builder.ins().iadd(*dst, *len); - *src = builder.ins().iadd(*src, *len); - has_chunk_branch( - builder, - &op, - backwards_chunk_block, - backwards_last_chunk_block, - ); - - // Execute the per-chunk backwards copy, adjusting pointers before - // the copy itself. - builder.switch_to_block(backwards_chunk_block); - append_block_params(builder, backwards_chunk_block, &mut op); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - let remaining_len = *len; - *len = chunk; - *dst = builder.ins().isub(*dst, chunk); - *src = builder.ins().isub(*src, chunk); - raw_call(self, builder, &op); - let BulkOp::MemoryCopy { len, .. } = &mut op else { - unreachable!() - }; - *len = builder.ins().isub(remaining_len, chunk); - has_chunk_branch( - builder, - &op, - backwards_chunk_block, - backwards_last_chunk_block, - ); - builder.seal_block(backwards_chunk_block); - - // Final backwards chunk: adjust the dst/src to be their true base - // pointers and then delegate to `last_chunk_block` for the actual - // memcpy. - builder.switch_to_block(backwards_last_chunk_block); - builder.seal_block(backwards_last_chunk_block); - append_block_params(builder, backwards_last_chunk_block, &mut op); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - *dst = builder.ins().isub(*dst, *len); - *src = builder.ins().isub(*src, *len); - builder.ins().jump( - last_chunk_block, - &[(*dst).into(), (*src).into(), (*len).into()], - ); } - - // In the final block we know that the length of the operation is less - // than `chunk`. - builder.switch_to_block(last_chunk_block); - builder.seal_block(last_chunk_block); - append_block_params(builder, last_chunk_block, &mut op); - raw_call(self, builder, &op); + Ok(()) } /// Emits a generic "fill" of `entity` from `dst` for `len` elements, @@ -4021,6 +3715,9 @@ impl FuncEnvironment<'_> { /// `dst` and `len` values must be typed appropriately for `entity`. This /// will perform a bounds-check before actually executing the operation and /// then afterwards will perform the operation. + /// + /// Callers must invoke `pre_translate_bulk_op` before calling this method + /// to properly account for fuel/epoch checks for this bulk operation. fn translate_entity_fill( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4028,7 +3725,6 @@ impl FuncEnvironment<'_> { dst: ir::Value, val: ir::Value, len: ir::Value, - cost_per_unit: u8, ) -> WasmResult<()> { let entity = entity.into(); let idx_type = entity.index_type(self); @@ -4042,37 +3738,22 @@ impl FuncEnvironment<'_> { self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, idx_type); match entity { - CheckedEntity::Memory(_) => { - self.raw_bulk_memory_operation( - builder, - BulkOp::MemoryFill { - dst: raw_dst_addr, - val, - len: len_ptr, - fuel: BulkFuel { - cost_per_unit, - bytes_per_unit: 1, - }, - }, - ); - } - CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => { - self.emit_raw_array_or_table_fill( - builder, - entity, - raw_dst_addr, + CheckedEntity::Memory(_) => self.raw_bulk_memory_operation( + builder, + BulkOp::MemoryFill { + dst: raw_dst_addr, val, - len_ptr, - cost_per_unit, - )?; + len: len_ptr, + }, + ), + CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => { + self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr) } // Not allowed to be written to in wasm. CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { unreachable!() } } - - Ok(()) } /// Performs a manual element-by-element fill of `entity`, starting at @@ -4096,7 +3777,6 @@ impl FuncEnvironment<'_> { dst_elem_addr: ir::Value, value: ir::Value, copy_len: ir::Value, - cost_per_element: u8, ) -> WasmResult<()> { let pointer_ty = self.pointer_type(); @@ -4116,19 +3796,14 @@ impl FuncEnvironment<'_> { if entity.allows_memset(self) && let Some(value) = self.fill_value_as_memset(builder, elem_ty, value) { - self.raw_bulk_memory_operation( + return self.raw_bulk_memory_operation( builder, BulkOp::MemoryFill { dst: dst_elem_addr, val: value, len: copy_byte_len, - fuel: BulkFuel { - cost_per_unit: cost_per_element, - bytes_per_unit: u8::try_from(elem_size).unwrap(), - }, }, ); - return Ok(()); } // Funcref values are intern'd when stored on the GC heap, and the @@ -4169,12 +3844,6 @@ impl FuncEnvironment<'_> { builder.insert_block_after(loop_block, current_block); builder.insert_block_after(continue_block, loop_block); - // Before entering the loop below flush our fuel counters to ensure - // that previous instructions' fuel isn't counted once-per-iteration. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - // Current block: test to see if this is actually an empty copy. If it // is then skip over the entire loop, otherwise enter the loop and // perform the first ieration. @@ -4192,11 +3861,6 @@ impl FuncEnvironment<'_> { // by the element size, then see if we turn again or exit. builder.switch_to_block(loop_block); let elem_addr = builder.append_block_param(loop_block, pointer_ty); - // Consume the configured cost for this element before writing it. - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; match entity { CheckedEntity::Table { table, initialized } => { assert!(!is_pre_interned_funcref); @@ -4312,7 +3976,8 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let cost = self.tunables.operator_cost.variable().memory_fill_per_byte; - self.translate_entity_fill(builder, memory_index, dst, val, len, cost) + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_fill(builder, memory_index, dst, val, len) } pub fn translate_memory_init( @@ -4326,6 +3991,7 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let seg_index = DataIndex::from_u32(seg_index); let cost = self.tunables.operator_cost.variable().memory_init_per_byte; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, memory_index, @@ -4336,7 +4002,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -4379,6 +4044,9 @@ impl FuncEnvironment<'_> { /// of the copy. Both `dst` and `src` have types appropriate to index their /// respective entities, and `len` has a type that's the smaller of the two /// index types. + /// + /// Callers must invoke `pre_translate_bulk_op` before calling this method + /// to properly account for fuel/epoch checks for this bulk operation. fn translate_entity_copy( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4387,7 +4055,6 @@ impl FuncEnvironment<'_> { dst: ir::Value, src: ir::Value, len: ir::Value, - cost_per_unit: u8, ) -> WasmResult<()> { let dst_entity = dst_entity.into(); let src_entity = src_entity.into(); @@ -4449,13 +4116,8 @@ impl FuncEnvironment<'_> { const_len: const_count, src_entity, dst_entity, - fuel: BulkFuel { - cost_per_unit, - bytes_per_unit: 1, - }, }, - ); - Ok(()) + ) } // Tables/arrays are sometimes a memcpy, sometimes a per-element @@ -4470,7 +4132,6 @@ impl FuncEnvironment<'_> { len_ptr, src, const_count, - cost_per_unit, ), // Cannot copy into a data or element segment in wasm. @@ -4686,6 +4347,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_copy_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -4699,7 +4361,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -4729,7 +4390,6 @@ impl FuncEnvironment<'_> { copy_len: ir::Value, src_index: ir::Value, const_count: Option, - cost_per_element: u8, ) -> WasmResult<()> { let pointer_type = self.pointer_type(); assert_eq!(builder.func.dfg.value_type(dst_elem_addr), pointer_type); @@ -4808,7 +4468,7 @@ impl FuncEnvironment<'_> { // parameters (or expand it inline; see `raw_bulk_memory_operation`). if !type_forbids_memcpy && dst_element_size == src_element_size { let const_len = const_count.and_then(|c| c.checked_mul(u64::from(dst_element_size))); - self.raw_bulk_memory_operation( + return self.raw_bulk_memory_operation( builder, BulkOp::MemoryCopy { dst: dst_elem_addr, @@ -4817,13 +4477,8 @@ impl FuncEnvironment<'_> { const_len, src_entity, dst_entity, - fuel: BulkFuel { - cost_per_unit: cost_per_element, - bytes_per_unit: u8::try_from(dst_element_size).unwrap(), - }, }, ); - return Ok(()); } // For other copies, this is a per-element loop. Use the helper to @@ -4837,7 +4492,6 @@ impl FuncEnvironment<'_> { src_elem_addr, copy_len, src_index, - cost_per_element, &|this, builder, dst, src, src_index| { let write_ty = dst_entity.storage_type(this); let val = match src_entity { @@ -5044,7 +4698,6 @@ impl FuncEnvironment<'_> { src_elem_addr: ir::Value, copy_len: ir::Value, src_index: ir::Value, - cost_per_element: u8, copy_one: &dyn Fn( &mut Self, &mut FunctionBuilder<'_>, @@ -5096,13 +4749,6 @@ impl FuncEnvironment<'_> { builder.insert_block_after(backwards_block, forward_block); builder.insert_block_after(done_block, backwards_block); - // Update our local fuel counter, if enabled, before entering the loops - // below. This zeros out `self.fuel_consumed` so we don't consume - // previous fuel on each iteration of the loop. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - // Terminate `current_block` by testing to see if we're copying any // elements at all. builder @@ -5192,11 +4838,6 @@ impl FuncEnvironment<'_> { let src_cur = builder.append_block_param(forward_block, self.pointer_type()); let src_index = builder.append_block_param(forward_block, src_index_ty); let forward_keepalives = append_keepalive_params(builder, forward_block); - // Consume the configured cost for this element before copying it. - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; copy_one(self, builder, dst_cur, src_cur, src_index)?; let dst_next = builder .ins() @@ -5222,10 +4863,6 @@ impl FuncEnvironment<'_> { let src_cur = builder.append_block_param(backwards_block, self.pointer_type()); let src_index = builder.append_block_param(backwards_block, src_index_ty); let backward_keepalives = append_keepalive_params(builder, backwards_block); - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; let dst_cur = { let size = builder .ins() @@ -5281,6 +4918,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_init_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -5291,7 +4929,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -5355,6 +4992,78 @@ impl FuncEnvironment<'_> { Ok(builder.ins().ireduce(ir::types::I32, ret)) } + /// Translation prefix before bulk operations such as `memory.copy`. + /// + /// Takes a dynamic value `units` for the size of the operation as well as + /// a `cost_per_unit` configured for this operation. If fuel is enabled + /// this fuel will be consumed, and if epochs are enabled then an epoch + /// check happens. If neither epochs nor fuel are enabled this is a noop. + fn pre_translate_bulk_op( + &mut self, + builder: &mut FunctionBuilder, + units: ir::Value, + cost_per_unit: u8, + ) -> WasmResult<()> { + let const_units = + Self::value_as_const_int(builder, units).map(|c| i64::try_from(c).unwrap_or(i64::MAX)); + + if self.tunables.consume_fuel && cost_per_unit > 0 { + match const_units { + // Fold constant costs directly into internal state. + Some(units) => { + self.fuel_consumed = self + .fuel_consumed + .saturating_add(units.saturating_mul(i64::from(cost_per_unit))) + } + + None => { + // Note that fuel is always a 64-bit counter. + // + // Also note that the cost is clamped to `i64::MAX` to + // prevent fuel counter overflows since `cost` is otherwise + // an untrusted value. + let units_clamped64 = match builder.func.dfg.value_type(units) { + ir::types::I32 => { + let units64 = builder.ins().uextend(ir::types::I64, units); + builder.ins().imul_imm_u(units64, i64::from(cost_per_unit)) + } + ir::types::I64 => { + let fuel = builder.ins().imul_imm_u(units, i64::from(cost_per_unit)); + let max = builder.ins().iconst(ir::types::I64, i64::MAX); + let max_units = builder + .ins() + .iconst(I64, i64::MAX / i64::from(cost_per_unit)); + let saturate = + builder + .ins() + .icmp(IntCC::UnsignedGreaterThan, units, max_units); + builder.ins().select(saturate, max, fuel) + } + _ => unreachable!(), + }; + self.fuel_increment_var(builder); + let fuel = builder.use_var(self.fuel_var); + let fuel = builder.ins().iadd(fuel, units_clamped64); + builder.def_var(self.fuel_var, fuel); + } + } + } + + // Skip explicit fuel/epoch checks for operations which are + // subjectively, and statically, considered cheap. + const SMALL_BULK_OP_COST: i64 = 128; + if let Some(units) = const_units + && let Some(cost) = units.checked_mul(i64::from(cost_per_unit)) + && cost <= SMALL_BULK_OP_COST + { + return Ok(()); + } + + // This isn't a loop header but for fuel/epoch purposes it's the same + // thing. + self.translate_loop_header(builder) + } + pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> { // Additionally if enabled check how much fuel we have remaining to see // if we've run out by this point. @@ -6233,6 +5942,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -6242,7 +5952,6 @@ impl FuncEnvironment<'_> { dst, val, len, - cost, ) } @@ -6278,7 +5987,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_init_per_element; - self.consume_variable_fuel(builder, segment_len, cost); + self.pre_translate_bulk_op(builder, segment_len, cost)?; // Re-use the `table.set` translation for making this a simple function // to define. That re-executes the bounds check which is a bit @@ -6351,7 +6060,8 @@ impl FuncEnvironment<'_> { let len = self.load_runtime_data_length(builder, data); let start = builder.ins().iconst(I32, 0); let cost = self.tunables.operator_cost.variable().memory_init_per_byte; - self.translate_entity_copy(builder, memory, data, offset, start, len, cost)?; + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_copy(builder, memory, data, offset, start, len)?; // Finalize control-flow for the `MemorySegmentOffset::Static` case // above. @@ -6521,7 +6231,6 @@ enum BulkOp { const_len: Option, src_entity: CheckedEntity, dst_entity: CheckedEntity, - fuel: BulkFuel, }, /// A `memory.fill` operation, setting all bytes of `dst` to `val`. @@ -6534,31 +6243,9 @@ enum BulkOp { dst: ir::Value, val: ir::Value, len: ir::Value, - fuel: BulkFuel, }, } -impl BulkOp { - fn len(&self) -> ir::Value { - match self { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => *len, - } - } - - fn fuel(&self) -> BulkFuel { - match self { - BulkOp::MemoryCopy { fuel, .. } | BulkOp::MemoryFill { fuel, .. } => *fuel, - } - } -} - -#[derive(Copy, Clone)] -struct BulkFuel { - cost_per_unit: u8, - /// Number of copied bytes represented by one billable unit. - bytes_per_unit: u8, -} - /// A list of entities which can participate in various kinds of bulk operations /// in wasm. /// diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index 183b0d1d1cd4..ac8275f027a9 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -801,9 +801,15 @@ pub fn translate_array_new( array_type_index: TypeIndex, elem: ir::Value, len: ir::Value, - cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})"); + let cost = func_env + .tunables + .operator_cost + .variable() + .array_new_per_element; + func_env.pre_translate_bulk_op(builder, len, cost)?; + let result = gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?; let zero = builder.ins().iconst(ir::types::I32, 0); @@ -818,7 +824,6 @@ pub fn translate_array_new( zero, elem, len, - cost_per_element, )?; log::trace!("translate_array_new(..) -> {result:?}"); Ok(result) @@ -829,9 +834,14 @@ pub fn translate_array_new_default( builder: &mut FunctionBuilder, array_type_index: TypeIndex, len: ir::Value, - cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new_default({array_type_index:?}, {len:?})"); + let cost = func_env + .tunables + .operator_cost + .variable() + .array_new_default_per_element; + func_env.pre_translate_bulk_op(builder, len, cost)?; let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index(); let array_ty = func_env.types.unwrap_array(interned_ty)?; @@ -850,7 +860,6 @@ pub fn translate_array_new_default( zero, elem, len, - cost_per_element, )?; Ok(result) } @@ -1754,8 +1763,10 @@ pub fn translate_array_new_entity( entity: CheckedEntity, entity_offset: ir::Value, len: ir::Value, - cost_per_element: u8, + cost_per_unit: u8, ) -> WasmResult { + env.pre_translate_bulk_op(builder, len, cost_per_unit)?; + // Before actually allocating this array first do a bounds-check on the // passive entity itself. let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index(); @@ -1774,7 +1785,6 @@ pub fn translate_array_new_entity( dst, entity_offset, len, - cost_per_element, )?; Ok(array) diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rs index 69d310a507c9..ea929cbb94cd 100644 --- a/crates/environ/src/component/translate/adapt.rs +++ b/crates/environ/src/component/translate/adapt.rs @@ -351,6 +351,9 @@ fn fact_import_to_core_def( fact::Import::Trap => simple_intrinsic(dfg::Trampoline::Trap), fact::Import::EnterSyncCall => simple_intrinsic(dfg::Trampoline::EnterSyncCall), fact::Import::ExitSyncCall => simple_intrinsic(dfg::Trampoline::ExitSyncCall), + fact::Import::UnsafeIntrinsic(intrinsic) => { + dfg::CoreDef::UnsafeIntrinsic(ty.unwrap_func().unwrap_module_type_index(), *intrinsic) + } } } diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rs index 302bc00162dd..4f2a0ae7f026 100644 --- a/crates/environ/src/fact.rs +++ b/crates/environ/src/fact.rs @@ -22,13 +22,13 @@ use crate::component::dfg::CoreDef; use crate::component::{ Adapter, AdapterOptions as AdapterOptionsDfg, CanonicalAbiInfo, ComponentTypesBuilder, FlatType, InterfaceType, RuntimeComponentInstanceIndex, StringEncoding, Transcode, - TypeFuncIndex, + TypeFuncIndex, UnsafeIntrinsic, }; use crate::fact::transcode::Transcoder; use crate::prelude::*; use crate::{ EntityRef, FuncIndex, GlobalIndex, IndexType, Memory, MemoryIndex, ModuleInternedTypeIndex, - PrimaryMap, Tunables, + PrimaryMap, Tunables, WasmValType, }; use std::collections::HashMap; use wasm_encoder::*; @@ -99,6 +99,9 @@ pub struct Module<'a> { imported_trap: Option, + /// Cached versions of unsafe intrinsics and where they were imported. + imported_unsafe_intrinsics: HashMap, + // Current status of index spaces from the imports generated so far. imported_funcs: PrimaryMap>, imported_memories: PrimaryMap, @@ -292,6 +295,7 @@ impl<'a> Module<'a> { imported_enter_sync_call: None, imported_exit_sync_call: None, imported_trap: None, + imported_unsafe_intrinsics: HashMap::new(), exports: Vec::new(), task_may_block: None, } @@ -773,6 +777,51 @@ impl<'a> Module<'a> { ) } + /// Imports the `context.get` intrinsic for the `slot`th context slot. + fn import_context_get(&mut self, slot: usize) -> FuncIndex { + let intrinsic = match slot { + 0 => UnsafeIntrinsic::ContextGetI32_0, + 1 => UnsafeIntrinsic::ContextGetI32_1, + _ => unreachable!(), + }; + self.import_unsafe_intrinsic(intrinsic, &format!("get{slot}")) + } + + /// Imports the `context.set` intrinsic for the `slot`th context slot. + fn import_context_set(&mut self, slot: usize) -> FuncIndex { + let intrinsic = match slot { + 0 => UnsafeIntrinsic::ContextSetI32_0, + 1 => UnsafeIntrinsic::ContextSetI32_1, + _ => unreachable!(), + }; + self.import_unsafe_intrinsic(intrinsic, &format!("set{slot}")) + } + + fn import_unsafe_intrinsic(&mut self, intrinsic: UnsafeIntrinsic, name: &str) -> FuncIndex { + let map = |ty: &WasmValType| match ty { + crate::WasmValType::I32 => ValType::I32, + crate::WasmValType::I64 => ValType::I64, + crate::WasmValType::F32 => ValType::F32, + crate::WasmValType::F64 => ValType::F64, + crate::WasmValType::V128 => ValType::V128, + crate::WasmValType::Ref(_) => unreachable!(), + }; + let params = intrinsic.core_params().iter().map(map).collect::>(); + let results = intrinsic.core_results().iter().map(map).collect::>(); + + self.import_simple_get_and_set( + "context", + name, + ¶ms, + &results, + Import::UnsafeIntrinsic(intrinsic), + |me| me.imported_unsafe_intrinsics.get(&intrinsic).copied(), + |me, idx| { + me.imported_unsafe_intrinsics.insert(intrinsic, idx); + }, + ) + } + fn translate_helper(&mut self, helper: Helper) -> FunctionId { *self.helper_funcs.entry(helper).or_insert_with(|| { // Generate a fresh `Function` with a unique id for what we're about to @@ -920,6 +969,8 @@ pub enum Import { /// An intrinsic used by FACT-generated modules to pop the task previously /// pushed by `EnterSyncCall`. ExitSyncCall, + /// An unsafe intrinsic, such as reading/writing `context.{get,set}` slots. + UnsafeIntrinsic(UnsafeIntrinsic), } impl Options { diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs index bf1236dd903c..52c451a33d75 100644 --- a/crates/environ/src/fact/trampoline.rs +++ b/crates/environ/src/fact/trampoline.rs @@ -30,7 +30,7 @@ use crate::fact::{ LinearMemoryOptions, Module, Options, }; use crate::prelude::*; -use crate::{FuncIndex, GlobalIndex, IndexType, Trap}; +use crate::{FuncIndex, GlobalIndex, IndexType, NUM_COMPONENT_CONTEXT_SLOTS, Trap}; use std::collections::HashMap; use std::mem; use std::ops::Range; @@ -884,6 +884,15 @@ impl<'a, 'b> Compiler<'a, 'b> { } result_locals.reverse(); + // The `exit-sync-call` intrinsic below will clobber this task's context + // slots, but if we've got a post-return we'll want to restore them + // temporarily for that. Save them if it's necessary. + let callee_context = if adapter.lift.post_return.is_some() { + self.save_context() + } else { + Vec::new() + }; + // Handle a few things related to the concurrent task infrastructure // after the callee has finished, such as: // @@ -916,11 +925,19 @@ impl<'a, 'b> Compiler<'a, 'b> { // And finally post-return state is handled here once all results/etc // are all translated. + // + // Note that for this call the callee's previous context is shuffled + // in-and-then-back-out after the call. if let Some(func) = adapter.lift.post_return { + let caller_context = self.save_context(); + self.restore_context(callee_context); for (result, _) in result_locals.iter() { self.instruction(LocalGet(*result)); } self.instruction(Call(func.as_u32())); + self.restore_context(caller_context); + } else { + assert!(callee_context.is_empty()); } for tmp in temps { @@ -3715,7 +3732,7 @@ impl<'a, 'b> Compiler<'a, 'b> { self.ptr_uconst(mem_opts, 0); self.ptr_uconst(mem_opts, align); self.alloc_size(mem_opts, &size); - self.instruction(Call(realloc.as_u32())); + self.call_realloc(realloc); let addr = self.local_set_new_tmp(mem_opts.ptr()); self.memory_operand(opts, addr, size, align, oob_trap) } @@ -3744,7 +3761,7 @@ impl<'a, 'b> Compiler<'a, 'b> { self.alloc_size(mem_opts, &prev_size); self.ptr_uconst(mem_opts, align); self.alloc_size(mem_opts, &size); - self.instruction(Call(realloc.as_u32())); + self.call_realloc(realloc); self.instruction(LocalSet(ptr.idx)); self.validate_guest_pointer(opts, &ptr, &size, align, oob_trap) } @@ -3925,6 +3942,56 @@ impl<'a, 'b> Compiler<'a, 'b> { local.needs_free = false; } + /// Reads all of the current task's `context.{get,set}` slots into fresh + /// temporary locals which can later be handed to `restore_context`. + fn save_context(&mut self) -> Vec { + if !self.module.tunables.concurrency_support { + return Vec::new(); + } + let mut saved = Vec::new(); + for slot in 0..NUM_COMPONENT_CONTEXT_SLOTS { + let get = self.module.import_context_get(slot); + self.instruction(Call(get.as_u32())); + saved.push(self.local_set_new_tmp(ValType::I32)); + } + saved + } + + /// Stores zero into all of the current task's `context.{get,set}` slots. + fn clear_context(&mut self) { + if !self.module.tunables.concurrency_support { + return; + } + for slot in 0..NUM_COMPONENT_CONTEXT_SLOTS { + let set = self.module.import_context_set(slot); + self.instruction(I32Const(0)); + self.instruction(Call(set.as_u32())); + } + } + + /// Stores the slot values previously read by `save_context` back into the + /// current task's `context.{get,set}` slots. + fn restore_context(&mut self, saved: Vec) { + for (slot, local) in saved.into_iter().enumerate() { + let set = self.module.import_context_set(slot); + self.instruction(LocalGet(local.idx)); + self.instruction(Call(set.as_u32())); + self.free_temp_local(local); + } + } + + /// Emits a call to a guest `realloc` function. + /// + /// Note that this has special handling of the current task's + /// `context.{get,set}` slots, namely they're saved/restored around this + /// call and zero'd out during the call. + fn call_realloc(&mut self, realloc: FuncIndex) { + let saved = self.save_context(); + self.clear_context(); + self.instruction(Call(realloc.as_u32())); + self.restore_context(saved); + } + fn instruction(&mut self, instr: Instruction) { instr.encode(&mut self.code); } diff --git a/crates/test-programs/src/bin/p2_cli_stdin_eisdir.rs b/crates/test-programs/src/bin/p2_cli_stdin_eisdir.rs new file mode 100644 index 000000000000..0538db3f7c4d --- /dev/null +++ b/crates/test-programs/src/bin/p2_cli_stdin_eisdir.rs @@ -0,0 +1,36 @@ +//! Guest program that reads from stdin expecting an `IsADirectory` error. +//! +//! When stdin is redirected from a directory (e.g. `< /some/dir`), the host +//! should surface the error as `StreamError::LastOperationFailed` with the +//! original `io::Error` preserved, recoverable via `filesystem-error-code`. + +use test_programs::wasi::cli::stdin; +use test_programs::wasi::filesystem::types::{self as filesystem, ErrorCode}; +use test_programs::wasi::io::streams::StreamError; + +fn main() { + let stdin = stdin::get_stdin(); + + // Keep polling until data or an error is available. + loop { + stdin.subscribe().block(); + match stdin.read(1024) { + Ok(bytes) if bytes.is_empty() => continue, + Ok(_) => panic!("expected an error reading from a directory, got data"), + Err(StreamError::Closed) => { + panic!("expected LastOperationFailed(IsDirectory), got Closed") + } + Err(StreamError::LastOperationFailed(err)) => { + // Use filesystem-error-code to recover the specific error. + let code = filesystem::filesystem_error_code(&err); + assert_eq!( + code, + Some(ErrorCode::IsDirectory), + "expected IsDirectory, got {code:?}" + ); + eprintln!("got expected ErrorCode::IsDirectory"); + return; + } + } + } +} diff --git a/crates/test-programs/src/bin/p2_cli_stdout_epipe.rs b/crates/test-programs/src/bin/p2_cli_stdout_epipe.rs new file mode 100644 index 000000000000..3e52fda4855d --- /dev/null +++ b/crates/test-programs/src/bin/p2_cli_stdout_epipe.rs @@ -0,0 +1,29 @@ +//! Guest program that writes to stdout until the pipe is closed, then verifies +//! it gets `StreamError::Closed` (which maps to EPIPE) rather than a trap or +//! generic EIO. + +use test_programs::wasi::cli::stdout; +use test_programs::wasi::io::streams::StreamError; + +fn main() { + let stdout = stdout::get_stdout(); + let chunk = vec![b'x'; 4096]; + + loop { + match stdout.blocking_write_and_flush(&chunk) { + Ok(()) => continue, + Err(StreamError::Closed) => { + // This is the expected outcome: the pipe was closed by the + // reader, and wasmtime correctly reports it as Closed (EPIPE). + eprintln!("got expected StreamError::Closed"); + return; + } + Err(StreamError::LastOperationFailed(err)) => { + panic!( + "unexpected LastOperationFailed (should have been Closed): {}", + err.to_debug_string() + ); + } + } + } +} diff --git a/crates/test-programs/src/bin/p2_file_read_write.rs b/crates/test-programs/src/bin/p2_file_read_write.rs index 8c39d6de573c..0c2b82045c87 100644 --- a/crates/test-programs/src/bin/p2_file_read_write.rs +++ b/crates/test-programs/src/bin/p2_file_read_write.rs @@ -5,6 +5,11 @@ fn main() { let preopens = wasi::filesystem::preopens::get_directories(); let (dir, _) = &preopens[0]; + assert_eq!( + dir.read_via_stream(0).err(), + Some(wasi::filesystem::types::ErrorCode::IsDirectory) + ); + let filename = "test.txt"; let file = dir .open_at( diff --git a/crates/test-programs/src/bin/p3_file_write_chunked.rs b/crates/test-programs/src/bin/p3_file_write_chunked.rs new file mode 100644 index 000000000000..d294206b8b3e --- /dev/null +++ b/crates/test-programs/src/bin/p3_file_write_chunked.rs @@ -0,0 +1,85 @@ +use futures::join; +use test_programs::p3::wasi::filesystem::types::{ + Descriptor, DescriptorFlags, OpenFlags, PathFlags, +}; +use test_programs::p3::{wasi, wit_stream}; +use wit_bindgen::StreamResult; + +struct Component; + +test_programs::p3::export!(Component); + +impl test_programs::p3::exports::wasi::cli::run::Guest for Component { + async fn run() -> Result<(), ()> { + let preopens = wasi::filesystem::preopens::get_directories(); + let (dir, _) = &preopens[0]; + test_chunked_write(dir, "chunked_write.txt").await; + Ok(()) + } +} + +fn bytes(offset: &mut usize, len: usize) -> Vec { + let mut buf = Vec::with_capacity(len); + for i in 0..len { + buf.push(((*offset + i) % 251) as u8); + } + *offset += len; + buf +} + +async fn test_chunked_write(dir: &Descriptor, filename: &str) { + let mut len = 16; + let mut pos = 0; + + let file = dir + .open_at( + PathFlags::empty(), + filename.to_string(), + OpenFlags::CREATE, + DescriptorFlags::READ | DescriptorFlags::WRITE, + ) + .await + .expect("creating a file for writing"); + + let (mut tx, rx) = wit_stream::new(); + join! { + async { + file.write_via_stream(rx, 0).await.unwrap(); + }, + async { + loop { + // Wasmtime shouldn't buffer this much data by default on the + // host, something should have done a short write earlier. + assert!(len <= 128 << 20); + let (result, remaining) = tx.write(bytes(&mut pos, len)).await; + assert!(matches!(result, StreamResult::Complete(_)), "bad result {result:?}"); + if remaining.remaining() == 0 { + len = len.checked_mul(2).unwrap(); + } else { + pos -= remaining.remaining(); + break; + } + } + drop(tx); + }, + }; + + let expected = bytes(&mut 0, pos); + let (rx, result) = file.read_via_stream(0); + let read_back = rx.collect().await; + result.await.unwrap(); + + assert_eq!( + read_back.len(), + expected.len(), + "wrong number of bytes read back" + ); + assert!( + read_back == expected, + "contents differ after a chunked write" + ); +} + +fn main() { + unreachable!() +} diff --git a/crates/test-programs/src/bin/p3_filesystem_file_read_write.rs b/crates/test-programs/src/bin/p3_filesystem_file_read_write.rs index b4a9518cbeb9..2559a8f83bc9 100644 --- a/crates/test-programs/src/bin/p3_filesystem_file_read_write.rs +++ b/crates/test-programs/src/bin/p3_filesystem_file_read_write.rs @@ -1,5 +1,7 @@ use futures::join; -use test_programs::p3::wasi::filesystem::types::{DescriptorFlags, OpenFlags, PathFlags}; +use test_programs::p3::wasi::filesystem::types::{ + DescriptorFlags, ErrorCode, OpenFlags, PathFlags, +}; use test_programs::p3::{wasi, wit_stream}; struct Component; @@ -11,6 +13,13 @@ impl test_programs::p3::exports::wasi::cli::run::Guest for Component { let preopens = wasi::filesystem::preopens::get_directories(); let (dir, _) = &preopens[0]; + let (_data_rx, data_fut) = dir.read_via_stream(0); + let err = data_fut.await.expect_err("directory read should fail"); + assert!( + matches!(err, ErrorCode::IsDirectory), + "unexpected error: {err:?}" + ); + let filename = "test.txt"; { let file = dir diff --git a/crates/test-programs/src/bin/p3_http_outbound_request_chunk_size.rs b/crates/test-programs/src/bin/p3_http_outbound_request_chunk_size.rs new file mode 100644 index 000000000000..7b7043da2b72 --- /dev/null +++ b/crates/test-programs/src/bin/p3_http_outbound_request_chunk_size.rs @@ -0,0 +1,86 @@ +use futures::join; +use test_programs::p3::wasi::http::client; +use test_programs::p3::wasi::http::types::{Headers, Method, Request, Response, Scheme}; +use test_programs::p3::{wit_future, wit_stream}; +use wit_bindgen::StreamResult; + +struct Component; + +test_programs::p3::export!(Component); + +fn bytes(offset: &mut usize, len: usize) -> Vec { + let mut buf = Vec::with_capacity(len); + for i in 0..len { + buf.push(((*offset + i) % 251) as u8); + } + *offset += len; + buf +} + +fn addr() -> String { + test_programs::p3::wasi::cli::environment::get_environment() + .into_iter() + .find_map(|(k, v)| k.eq("HTTP_SERVER").then_some(v)) + .unwrap() +} + +impl test_programs::p3::exports::wasi::cli::run::Guest for Component { + async fn run() -> Result<(), ()> { + test_chunked_write().await; + Ok(()) + } +} + +async fn test_chunked_write() { + let headers = Headers::from_list(&[]).unwrap(); + let (mut contents_tx, contents_rx) = wit_stream::new(); + let (trailers_tx, trailers_rx) = wit_future::new(|| Ok(None)); + let (request, transmit) = Request::new(headers, Some(contents_rx), trailers_rx, None); + configure(&request); + + let (transmit, written, echoed) = join!( + async { transmit.await }, + async { + let mut len = 16; + let mut pos = 0; + loop { + assert!(len <= 128 << 20); + let (result, remaining) = contents_tx.write(bytes(&mut pos, len)).await; + assert_eq!(result, StreamResult::Complete(len - remaining.remaining())); + if remaining.remaining() == 0 { + len = len.checked_mul(2).unwrap(); + } else { + pos -= remaining.remaining(); + break; + } + } + drop(contents_tx); + _ = trailers_tx.write(Ok(None)).await; + pos + }, + async { send_and_collect(request).await }, + ); + transmit.unwrap(); + assert_eq!(echoed, bytes(&mut 0, written)); +} + +fn configure(request: &Request) { + request.set_method(&Method::Post).unwrap(); + request.set_scheme(Some(&Scheme::Http)).unwrap(); + request.set_authority(Some(&addr())).unwrap(); + request.set_path_with_query(Some("/")).unwrap(); +} + +async fn send_and_collect(request: Request) -> Vec { + let response = client::send(request).await.unwrap(); + assert_eq!(response.get_status_code(), 200); + let (_, result_rx) = wit_future::new(|| Ok(())); + let (body_rx, trailers_rx) = Response::consume_body(response, result_rx); + let body = body_rx.collect().await; + trailers_rx.await.unwrap(); + body +} + +fn main() { + unreachable!() +} diff --git a/crates/wasi-http/src/ctx.rs b/crates/wasi-http/src/ctx.rs index c5baf3b33a9a..2d58c1e771d8 100644 --- a/crates/wasi-http/src/ctx.rs +++ b/crates/wasi-http/src/ctx.rs @@ -347,6 +347,13 @@ pub trait WasiHttpHooks: Send { }) } + /// Maximum number of bytes the implementation will copy out of the guest in + /// a single write to an outgoing body's stream. + #[cfg(feature = "p3")] + fn p3_outgoing_body_chunk_size(&mut self) -> usize { + crate::p3::DEFAULT_OUTGOING_BODY_CHUNK_SIZE + } + /// Optional hook to configure the error code for hyper errors. #[cfg(feature = "p3")] fn p3_error_from_hyper(&mut self, err: &hyper::Error) -> p3::ErrorCode { diff --git a/crates/wasi-http/src/p2/http_impl.rs b/crates/wasi-http/src/p2/http_impl.rs index 8ae075eadb14..d0d1ccedbda6 100644 --- a/crates/wasi-http/src/p2/http_impl.rs +++ b/crates/wasi-http/src/p2/http_impl.rs @@ -44,12 +44,25 @@ impl outgoing_handler::Host for WasiHttpCtxView<'_> { }, }); - let scheme = match req.scheme.unwrap_or(Scheme::Https) { - Scheme::Http => http::uri::Scheme::HTTP, - Scheme::Https => http::uri::Scheme::HTTPS, - - // We can only support http/https - Scheme::Other(_) => return Err(types::ErrorCode::HttpProtocolError.into()), + let scheme = match req.scheme { + Some(scheme) => { + let scheme = match scheme { + Scheme::Http => http::uri::Scheme::HTTP, + Scheme::Https => http::uri::Scheme::HTTPS, + Scheme::Other(scheme) => http::uri::Scheme::try_from(scheme.as_str()) + .map_err(|_| types::ErrorCode::HttpProtocolError)?, + }; + if !self.hooks.is_supported_scheme(&scheme) { + return Err(types::ErrorCode::HttpProtocolError.into()); + } + scheme + } + // Note that a hook returning `None` here means that guests are + // required to specify a scheme themselves. + None => self + .hooks + .default_scheme() + .ok_or(types::ErrorCode::HttpProtocolError)?, }; let authority = req.authority.unwrap_or_else(String::new); @@ -64,6 +77,10 @@ impl outgoing_handler::Host for WasiHttpCtxView<'_> { builder = builder.uri(uri.build().map_err(http_request_error)?); + if self.hooks.set_host_header() { + builder = builder.header(http::header::HOST, authority.as_str()); + } + for (k, v) in req.headers.iter() { builder = builder.header(k, v); } diff --git a/crates/wasi-http/src/p3/body.rs b/crates/wasi-http/src/p3/body.rs index b5795e86bfd1..4712aa9905f7 100644 --- a/crates/wasi-http/src/p3/body.rs +++ b/crates/wasi-http/src/p3/body.rs @@ -177,6 +177,7 @@ struct LimitedGuestBodyConsumer { limit: u64, /// Number of bytes sent sent: u64, + max_chunk_size: usize, // `true` when the other side of `contents_tx` was unexpectedly closed closed: bool, } @@ -217,7 +218,8 @@ impl StreamConsumer for LimitedGuestBodyConsumer { debug_assert!(!self.closed); let mut src = src.as_direct(store); let buf = src.remaining(); - let n = buf.len(); + let n = buf.len().min(self.max_chunk_size); + let buf = &buf[..n]; // Perform `content-length` check early and precompute the next value let Ok(sent) = n.try_into() else { @@ -260,7 +262,10 @@ impl StreamConsumer for LimitedGuestBodyConsumer { /// [StreamConsumer] implementation for bodies originating in the guest without `Content-Length` /// header set. -struct UnlimitedGuestBodyConsumer(PollSender>); +struct UnlimitedGuestBodyConsumer { + contents_tx: PollSender>, + max_chunk_size: usize, +} impl StreamConsumer for UnlimitedGuestBodyConsumer { type Item = u8; @@ -272,13 +277,13 @@ impl StreamConsumer for UnlimitedGuestBodyConsumer { src: Source, finish: bool, ) -> Poll> { - match self.0.poll_reserve(cx) { + match self.contents_tx.poll_reserve(cx) { Poll::Ready(Ok(())) => { let mut src = src.as_direct(store); let buf = src.remaining(); - let n = buf.len(); - let buf = Bytes::copy_from_slice(buf); - match self.0.send_item(Ok(buf)) { + let n = buf.len().min(self.max_chunk_size); + let buf = Bytes::copy_from_slice(&buf[..n]); + match self.contents_tx.send_item(Ok(buf)) { Ok(()) => { src.mark_read(n); Poll::Ready(Ok(StreamResult::Completed)) @@ -329,6 +334,11 @@ impl GuestBody { Ok(()) })?; + let max_chunk_size = getter(store.as_context_mut().data_mut()) + .hooks + .p3_outgoing_body_chunk_size() + .max(1); + let contents_rx = if let Some(rx) = contents_rx { let (http_tx, http_rx) = mpsc::channel(1); let contents_tx = PollSender::new(http_tx); @@ -348,12 +358,19 @@ impl GuestBody { make_error, limit, sent: 0, + max_chunk_size, closed: false, }, )?; } else { _ = result_tx.send(Box::new(result_fut)); - rx.pipe(store, UnlimitedGuestBodyConsumer(contents_tx))?; + rx.pipe( + store, + UnlimitedGuestBodyConsumer { + contents_tx, + max_chunk_size, + }, + )?; }; Some(http_rx) } else { diff --git a/crates/wasi-http/src/p3/mod.rs b/crates/wasi-http/src/p3/mod.rs index 3c6cfdf72513..39c8b197270a 100644 --- a/crates/wasi-http/src/p3/mod.rs +++ b/crates/wasi-http/src/p3/mod.rs @@ -20,6 +20,11 @@ mod response; pub use request::Request; pub use response::Response; +/// The default value configured for [`WasiHttpHooks::p3_outgoing_body_chunk_size`]. +/// +/// [`WasiHttpHooks::p3_outgoing_body_chunk_size`]: crate::WasiHttpHooks::p3_outgoing_body_chunk_size +pub const DEFAULT_OUTGOING_BODY_CHUNK_SIZE: usize = 1024 * 1024; + use crate::{FieldMapError, WasiHttp, WasiHttpView}; use bindings::http::{client, types}; use core::ops::Deref; diff --git a/crates/wasi-http/tests/all/p3/mod.rs b/crates/wasi-http/tests/all/p3/mod.rs index a97ef9d1c725..af68212259fb 100644 --- a/crates/wasi-http/tests/all/p3/mod.rs +++ b/crates/wasi-http/tests/all/p3/mod.rs @@ -127,6 +127,7 @@ async fn run_cli(path: &str, server: &Server) -> wasmtime::Result<()> { Ctx { wasi: wasmtime_wasi::WasiCtx::builder() .env("HTTP_SERVER", server.addr()) + .inherit_stdio() .build(), ..Ctx::new(oneshot::channel().0) }, @@ -870,3 +871,9 @@ async fn p3_http_empty_frames_interleaved() -> Result<()> { assert_eq!(collected_body, b"hello world".as_slice()); Ok(()) } + +#[test_log::test(tokio::test(flavor = "multi_thread"))] +async fn p3_http_outbound_request_chunk_size() -> Result<()> { + let server = Server::http1(1)?; + run_cli(P3_HTTP_OUTBOUND_REQUEST_CHUNK_SIZE_COMPONENT, &server).await +} diff --git a/crates/wasi/src/cli.rs b/crates/wasi/src/cli.rs index 090a656c4c76..eff5089d1cc6 100644 --- a/crates/wasi/src/cli.rs +++ b/crates/wasi/src/cli.rs @@ -3,7 +3,7 @@ use std::pin::Pin; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, empty}; use wasmtime::component::{HasData, ResourceTable}; -use wasmtime_wasi_io::streams::{InputStream, OutputStream}; +use wasmtime_wasi_io::streams::{InputStream, OutputStream, StreamError}; mod empty; mod file; @@ -15,6 +15,27 @@ mod worker_thread_stdin; pub use self::file::{InputFile, OutputFile}; pub use self::locked_async::{AsyncStdinStream, AsyncStdoutStream}; +/// Convert a host `io::Error` into a `StreamError`, matching the error-code +/// recovery that wasip1 performs via `filesystem::ErrorCode::from`. +/// +/// * `BrokenPipe` is mapped to `StreamError::Closed` so that downstream +/// consumers (e.g. wasi-libc) can recover `EPIPE` rather than falling back +/// to a generic `EIO`. +/// +/// * All other errors (including `IsADirectory`, permission errors, etc.) are +/// preserved as `LastOperationFailed` with the original `std::io::Error` +/// intact. This allows guests to recover the specific error code via the +/// `wasi:filesystem/types#filesystem-error-code` function, which downcasts +/// the error back to `std::io::Error` and maps it through +/// `ErrorCode::from`. +fn stream_error_from(e: std::io::Error) -> StreamError { + if e.kind() == std::io::ErrorKind::BrokenPipe { + StreamError::Closed + } else { + StreamError::LastOperationFailed(e.into()) + } +} + // Convenience reexport for stdio types so tokio doesn't have to be imported // itself. #[doc(no_inline)] @@ -366,4 +387,66 @@ mod test { s.write_ready().await?; Ok(()) } + + // Verify that the stdio OutputStream implementation reports a usable + // write permit and can successfully write + flush (exercises the full + // trait impl including the error conversion path). + #[test] + fn stdio_output_stream_write_flush() { + let mut stream: Box = + StdoutStream::p2_stream(&std::io::stderr()); + + let permit = stream.check_write().expect("check_write"); + assert!(permit > 0, "permit should be nonzero"); + + // Writing empty bytes must succeed. + stream + .write(Bytes::new()) + .expect("writing empty bytes should succeed"); + + // Flushing must succeed. + stream.flush().expect("flush should succeed"); + } + + #[test] + fn stream_error_from_broken_pipe_maps_to_closed() { + use std::io; + use wasmtime_wasi_io::streams::StreamError; + + let err = super::stream_error_from(io::Error::from(io::ErrorKind::BrokenPipe)); + assert!(matches!(err, StreamError::Closed)); + } + + #[test] + fn stream_error_from_preserves_io_error() { + use std::io; + use wasmtime_wasi_io::streams::StreamError; + + let err = super::stream_error_from(io::Error::from(io::ErrorKind::IsADirectory)); + match err { + StreamError::LastOperationFailed(e) => { + let io_err = e.downcast::().expect("should downcast"); + assert_eq!(io_err.kind(), io::ErrorKind::IsADirectory); + } + other => panic!("expected LastOperationFailed, got: {other:?}"), + } + } + + #[cfg(unix)] + #[test] + fn stream_error_from_raw_os_eisdir() { + use rustix::io::Errno; + use std::io; + use wasmtime_wasi_io::streams::StreamError; + + let err = + super::stream_error_from(io::Error::from_raw_os_error(Errno::ISDIR.raw_os_error())); + match err { + StreamError::LastOperationFailed(e) => { + let io_err = e.downcast::().expect("should downcast"); + assert_eq!(io_err.raw_os_error(), Some(Errno::ISDIR.raw_os_error())); + } + other => panic!("expected LastOperationFailed, got: {other:?}"), + } + } } diff --git a/crates/wasi/src/cli/stdout.rs b/crates/wasi/src/cli/stdout.rs index 9d0a213e528c..0ffe2ee976b3 100644 --- a/crates/wasi/src/cli/stdout.rs +++ b/crates/wasi/src/cli/stdout.rs @@ -1,4 +1,4 @@ -use crate::cli::{IsTerminal, StdoutStream}; +use crate::cli::{IsTerminal, StdoutStream, stream_error_from}; use crate::p2; use bytes::Bytes; use std::io::{self, Write}; @@ -78,7 +78,7 @@ impl OutputStream for StdioOutputStream { StdioOutputStream::Stdout => std::io::stdout().write_all(&bytes), StdioOutputStream::Stderr => std::io::stderr().write_all(&bytes), } - .map_err(|e| p2::StreamError::LastOperationFailed(wasmtime::format_err!(e))) + .map_err(|e| stream_error_from(e)) } fn flush(&mut self) -> p2::StreamResult<()> { @@ -86,7 +86,7 @@ impl OutputStream for StdioOutputStream { StdioOutputStream::Stdout => std::io::stdout().flush(), StdioOutputStream::Stderr => std::io::stderr().flush(), } - .map_err(|e| p2::StreamError::LastOperationFailed(wasmtime::format_err!(e))) + .map_err(|e| stream_error_from(e)) } fn check_write(&mut self) -> p2::StreamResult { diff --git a/crates/wasi/src/cli/worker_thread_stdin.rs b/crates/wasi/src/cli/worker_thread_stdin.rs index 6f92190ce1dd..06ee1cb07017 100644 --- a/crates/wasi/src/cli/worker_thread_stdin.rs +++ b/crates/wasi/src/cli/worker_thread_stdin.rs @@ -23,7 +23,7 @@ //! This module is one that's likely to change over time though as new systems //! are encountered along with preexisting bugs. -use crate::cli::{IsTerminal, StdinStream}; +use crate::cli::{IsTerminal, StdinStream, stream_error_from}; use bytes::{Bytes, BytesMut}; use std::io::Read; use std::mem; @@ -176,7 +176,7 @@ impl InputStream for WasiStdin { } StdinState::Error(e) => { *locked = StdinState::Closed; - Err(StreamError::LastOperationFailed(e.into())) + Err(stream_error_from(e)) } StdinState::Closed => { *locked = StdinState::Closed; diff --git a/crates/wasi/src/ctx.rs b/crates/wasi/src/ctx.rs index 13c350383001..99a8c71c3a9c 100644 --- a/crates/wasi/src/ctx.rs +++ b/crates/wasi/src/ctx.rs @@ -414,10 +414,9 @@ impl WasiCtxBuilder { self } - /// Allow usage of UDP. + /// Allow usage of UDP /// - /// This is enabled by default, but can be disabled if UDP should be blanket - /// disabled. + /// By default this is disabled. pub fn allow_udp(&mut self, enable: bool) -> &mut Self { self.sockets.allowed_network_uses.udp = enable; self @@ -425,8 +424,7 @@ impl WasiCtxBuilder { /// Allow usage of TCP /// - /// This is enabled by default, but can be disabled if TCP should be blanket - /// disabled. + /// By default this is disabled. pub fn allow_tcp(&mut self, enable: bool) -> &mut Self { self.sockets.allowed_network_uses.tcp = enable; self diff --git a/crates/wasi/src/filesystem.rs b/crates/wasi/src/filesystem.rs index a8e90bf9895e..e7b9190bb68c 100644 --- a/crates/wasi/src/filesystem.rs +++ b/crates/wasi/src/filesystem.rs @@ -490,6 +490,9 @@ impl Descriptor { pub(crate) fn file(&self) -> Result<&File, ErrorCode> { match self { Descriptor::File(f) => Ok(f), + // File-only ops such as advise stay bad-descriptor on a dir + // (wasi-testsuite filesystem-advise). read-via-stream maps Dir + // to is-directory on its own. Descriptor::Dir(_) => Err(ErrorCode::BadDescriptor), } } diff --git a/crates/wasi/src/p2/host/filesystem.rs b/crates/wasi/src/p2/host/filesystem.rs index 3fc37a45ad07..579c74129a05 100644 --- a/crates/wasi/src/p2/host/filesystem.rs +++ b/crates/wasi/src/p2/host/filesystem.rs @@ -367,8 +367,12 @@ impl HostDescriptor for WasiFilesystemCtxView<'_> { fd: Resource, offset: types::Filesize, ) -> FsResult> { - // Trap if fd lookup fails: - let f = self.table.get(&fd)?.file()?; + // Trap if fd lookup fails. A directory is is-directory, not + // bad-descriptor (POSIX EISDIR on read). + let f = match self.table.get(&fd)? { + Descriptor::File(f) => f, + Descriptor::Dir(_) => return Err(ErrorCode::IsDirectory.into()), + }; // Create a stream view for it. let reader: DynInputStream = Box::new(FileInputStream::new(f, offset)); diff --git a/crates/wasi/src/p3/filesystem/host.rs b/crates/wasi/src/p3/filesystem/host.rs index 54437ce23fa0..f71e61d810ae 100644 --- a/crates/wasi/src/p3/filesystem/host.rs +++ b/crates/wasi/src/p3/filesystem/host.rs @@ -463,7 +463,9 @@ impl StreamConsumer for WriteStreamConsumer { let me = &mut *self; let task = me.task.get_or_insert_with(|| { debug_assert!(me.buffer.is_empty()); - me.buffer.extend_from_slice(src.remaining()); + let remaining = src.remaining(); + let n = remaining.len().min(DEFAULT_BUFFER_CAPACITY); + me.buffer.extend_from_slice(&remaining[..n]); let buf = mem::take(&mut me.buffer); let file = Arc::clone(me.file.as_file()); let location = me.location; @@ -521,8 +523,17 @@ impl types::HostDescriptorWithStore for WasiFilesystem { fd: Resource, offset: Filesize, ) -> wasmtime::Result<(StreamReader, FutureReader>)> { - let file = get_file(store.get().table, &fd)?; - let file = file.clone(); + let file = match get_descriptor(store.get().table, &fd)? { + Descriptor::File(file) => file.clone(), + Descriptor::Dir(_) => { + return Ok(( + StreamReader::new(&mut store, iter::empty())?, + FutureReader::new(&mut store, async move { + wasmtime::error::Ok(Err(ErrorCode::IsDirectory)) + })?, + )); + } + }; let (result_tx, result_rx) = oneshot::channel(); Ok(( StreamReader::new( diff --git a/crates/wasi/tests/all/p3/mod.rs b/crates/wasi/tests/all/p3/mod.rs index a00dc7d64db7..4cf311588322 100644 --- a/crates/wasi/tests/all/p3/mod.rs +++ b/crates/wasi/tests/all/p3/mod.rs @@ -185,6 +185,11 @@ async fn p3_file_write_blocking() -> wasmtime::Result<()> { run_allow_blocking_current_thread(P3_FILE_WRITE_COMPONENT, true).await } +#[test_log::test(tokio::test(flavor = "multi_thread"))] +async fn p3_file_write_chunked() -> wasmtime::Result<()> { + run(P3_FILE_WRITE_CHUNKED_COMPONENT).await +} + #[test_log::test(tokio::test(flavor = "multi_thread"))] async fn p3_file_truncation_readonly() -> wasmtime::Result<()> { run_with_readonly_testfile(P3_FILE_TRUNCATION_READONLY_COMPONENT).await diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 6534da5b5eab..c833bc1a8d79 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -690,10 +690,6 @@ impl Config { /// signal handler), then we can ensure that all async code will /// yield to the executor within a bounded time. /// - /// The deadline check cannot be avoided by malicious wasm code. It is safe - /// to use epoch deadlines to limit the execution time of untrusted - /// code. - /// /// The [`Store`](crate::Store) tracks the deadline, and controls /// what happens when the deadline is reached during /// execution. Several behaviors are possible: @@ -739,6 +735,27 @@ impl Config { /// computation and have the desired effect of cancelling a blocking /// operation when a timeout expires. /// + /// ## Limitations with malicious guests + /// + /// Epochs are designed to handle malicious WebAssembly guests -- the + /// deadline check cannot be avoided by WebAssembly code. It is safe to use + /// epoch deadlines to limit the execution time of untrusted code. + /// + /// Note, though, that a current limitation to this is that + /// bulk-data-transfer instructions, such as `memory.copy`, only check the + /// epoch once at the start of the operation. These operations can take a + /// variable amount of time to complete based on how many bytes are being + /// copied. This means that the maximal time slice a guest might take is + /// the maximum of the epoch interval and the largest + /// memory-copy-style-instruction executed. The size of a copy is bounded + /// on the size of linear memory or GC heap size. In the limit, however, a + /// guest using a 64-bit linear memory with a 128GiB size could issue a + /// 128GiB `memory.copy` which would have no preemption within the + /// instruction itself. Hosts which need strict time limits for guests right + /// now are recommended to ensure that the store's allocated heap size + /// (linear memory + GC heap) are bounded with a + /// [`ResourceLimiter`](crate::ResourceLimiter). + /// /// ## When to use fuel vs. epochs /// /// In general, epoch-based interruption results in faster diff --git a/crates/wasmtime/src/runtime/component/linker.rs b/crates/wasmtime/src/runtime/component/linker.rs index 7ab67228533c..cb41df20320f 100644 --- a/crates/wasmtime/src/runtime/component/linker.rs +++ b/crates/wasmtime/src/runtime/component/linker.rs @@ -18,10 +18,10 @@ use wasmtime_environ::{Atom, PrimaryMap, StringPool}; /// A type used to instantiate [`Component`]s. /// -/// This type is used to both link components together as well as supply host -/// functionality to components. Values are defined in a [`Linker`] by their -/// import name and then components are instantiated with a [`Linker`] using the -/// names provided for name resolution of the component's imports. +/// This type is used to supply host functionality to components. Values are +/// defined in a [`Linker`] by their import name and then components are +/// instantiated with a [`Linker`] using the names provided for name resolution +/// of the component's imports. /// /// # Names and Semver /// diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 9cc49ea8361c..a0816548b844 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1041,6 +1041,10 @@ impl Store { /// The `interval` parameter indicates how much fuel should be /// consumed between yields of an async future. When fuel runs out wasm will trap. /// + /// For limitations related to consumption of fuel and when yield points are + /// injected, see the discussion in + /// [`Config::epoch_interruption`](crate::Config::epoch_interruption). + /// /// # Error /// /// This method will error if fuel is not enabled or `interval` is diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rs index 6cc53eff0092..398d86f9352a 100644 --- a/crates/wasmtime/src/runtime/vm/cow.rs +++ b/crates/wasmtime/src/runtime/vm/cow.rs @@ -4,6 +4,7 @@ use super::sys::DecommitBehavior; use crate::Engine; use crate::prelude::*; +use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::sys::vm::{self, MemoryImageSource, PageMap, reset_with_pagemap}; use crate::runtime::vm::{ HostAlignedByteCount, MmapOffset, ModuleMemoryImageSource, host_page_size, @@ -337,6 +338,14 @@ pub struct MemoryImageSlot { /// initial image content, as appropriate. Everything between /// `self.accessible` and `self.static_size` is inaccessible. dirty: bool, + + /// The MPK protection key that this slot's stripe was colored with, if the + /// pooling allocator is striping memory with protection keys. + /// + /// This must be re-applied after every `mmap` performed on this slot, since + /// `mmap` resets the affected pages back to the default key 0 which is + /// accessible from every stripe. + pkey: Option, } impl fmt::Debug for MemoryImageSlot { @@ -363,6 +372,7 @@ impl MemoryImageSlot { base: MmapOffset, accessible: HostAlignedByteCount, static_size: usize, + pkey: Option, ) -> Self { MemoryImageSlot { base, @@ -370,6 +380,7 @@ impl MemoryImageSlot { accessible, image: None, dirty: false, + pkey, } } @@ -489,6 +500,11 @@ impl MemoryImageSlot { unsafe { image.map_at(&self.base)?; } + // `map_at` above `mmap`'d over part of this slot, which + // reset those pages to the default protection key. Restore + // this slot's key so the image is not left accessible to + // every other stripe. + self.reapply_pkey(image.linear_memory_offset, image.len, true)?; } } self.image = maybe_image.cloned(); @@ -506,6 +522,9 @@ impl MemoryImageSlot { unsafe { image.remap_as_zeros_at(self.base.as_mut_ptr())?; } + // As in `instantiate`, the `mmap` above dropped this slot's + // protection key over the image's range, so restore it. + self.reapply_pkey(image.linear_memory_offset, image.len, true)?; self.image = None; } Ok(()) @@ -682,6 +701,43 @@ impl MemoryImageSlot { Ok(()) } + /// Re-color `offset..offset + len` within this slot with this slot's MPK + /// protection key, if any. + /// + /// This is a no-op unless the pooling allocator is striping memory with + /// protection keys. It must be called after every `mmap` that lands inside + /// this slot: `mmap` associates the pages it replaces with the default key + /// 0, which is accessible regardless of which stripe is currently active, + /// so skipping this would let one instance read and write another + /// instance's memory. + /// + /// Note that `mprotect` preserves the existing key, so `set_protection` + /// does not need this treatment. + fn reapply_pkey( + &self, + offset: HostAlignedByteCount, + len: HostAlignedByteCount, + readwrite: bool, + ) -> Result<()> { + let Some(pkey) = self.pkey else { + return Ok(()); + }; + if len.is_zero() { + return Ok(()); + } + // `mmap` rounds lengths up to a page boundary, so the restored range is + // allowed to extend to the end of the slot's final page. + debug_assert!( + offset.byte_count() + len.byte_count() + <= self.static_size.next_multiple_of(host_page_size()) + ); + unsafe { + let start = self.base.as_mut_ptr().add(offset.byte_count()); + pkey.reprotect(start.addr(), len.byte_count(), readwrite)?; + } + Ok(()) + } + pub(crate) fn has_image(&self) -> bool { self.image.is_some() } @@ -704,6 +760,11 @@ impl MemoryImageSlot { vm::erase_existing_mapping(self.base.as_mut_ptr(), self.static_size)?; } + // The `mmap` above covers the whole slot and left it inaccessible, so + // restore this slot's protection key across the same range. + let static_size = HostAlignedByteCount::new_rounded_up(self.static_size)?; + self.reapply_pkey(HostAlignedByteCount::ZERO, static_size, false)?; + self.image = None; self.accessible = HostAlignedByteCount::ZERO; @@ -813,8 +874,12 @@ mod test { // 4 MiB mmap'd area, not accessible let mmap = mmap_4mib_inaccessible(); // Create a MemoryImageSlot on top of it - let mut memfd = - MemoryImageSlot::create(mmap.zero_offset(), HostAlignedByteCount::ZERO, 4 << 20); + let mut memfd = MemoryImageSlot::create( + mmap.zero_offset(), + HostAlignedByteCount::ZERO, + 4 << 20, + None, + ); assert!(!memfd.is_dirty()); // instantiate with 64 KiB initial size memfd @@ -872,8 +937,12 @@ mod test { // 4 MiB mmap'd area, not accessible let mmap = mmap_4mib_inaccessible(); // Create a MemoryImageSlot on top of it - let mut memfd = - MemoryImageSlot::create(mmap.zero_offset(), HostAlignedByteCount::ZERO, 4 << 20); + let mut memfd = MemoryImageSlot::create( + mmap.zero_offset(), + HostAlignedByteCount::ZERO, + 4 << 20, + None, + ); // Create an image with some data. let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap()); // Instantiate with this image @@ -983,8 +1052,12 @@ mod test { ..Tunables::default_miri() }; let mmap = mmap_4mib_inaccessible(); - let mut memfd = - MemoryImageSlot::create(mmap.zero_offset(), HostAlignedByteCount::ZERO, 4 << 20); + let mut memfd = MemoryImageSlot::create( + mmap.zero_offset(), + HostAlignedByteCount::ZERO, + 4 << 20, + None, + ); // Test basics with the image for image_off in [0, page_size, page_size * 2] { @@ -1057,8 +1130,12 @@ mod test { }; let mmap = mmap_4mib_inaccessible(); - let mut memfd = - MemoryImageSlot::create(mmap.zero_offset(), HostAlignedByteCount::ZERO, 4 << 20); + let mut memfd = MemoryImageSlot::create( + mmap.zero_offset(), + HostAlignedByteCount::ZERO, + 4 << 20, + None, + ); let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap()); let initial = 64 << 10; @@ -1166,8 +1243,12 @@ mod test { }; let mmap = mmap_4mib_inaccessible(); let mmap_len = page_size * 9; - let mut memfd = - MemoryImageSlot::create(mmap.zero_offset(), HostAlignedByteCount::ZERO, mmap_len); + let mut memfd = MemoryImageSlot::create( + mmap.zero_offset(), + HostAlignedByteCount::ZERO, + mmap_len, + None, + ); let pagemap = PageMap::new(); let pagemap = pagemap.as_ref(); diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs index 1dc57b12324e..5a554d72cd7a 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs @@ -589,6 +589,19 @@ impl MemoryPool { self.mapping.offset(offset).expect("offset is in bounds") } + /// Return the protection key that this slot's memory was striped with when + /// the pool was created, if any. + /// + /// This mirrors the striping performed in `new`: memory is only colored + /// when there are at least two stripes, and slot `i` is colored with the + /// `i % num_stripes`th key. + fn pkey_for_slot(&self, allocation_index: MemoryAllocationIndex) -> Option { + if self.stripes.len() < 2 { + return None; + } + self.stripes[allocation_index.index() % self.stripes.len()].pkey + } + /// Take ownership of the given image slot. /// /// This method is used when a `MemoryAllocationIndex` has been allocated @@ -620,6 +633,7 @@ impl MemoryPool { self.get_base(allocation_index), HostAlignedByteCount::ZERO, self.layout.max_memory_bytes.byte_count(), + self.pkey_for_slot(allocation_index), ) }); diff --git a/crates/wasmtime/src/runtime/vm/memory.rs b/crates/wasmtime/src/runtime/vm/memory.rs index fce3c4afa03f..ef68214ed745 100644 --- a/crates/wasmtime/src/runtime/vm/memory.rs +++ b/crates/wasmtime/src/runtime/vm/memory.rs @@ -570,8 +570,10 @@ impl LocalMemory { } }; + // Memories allocated on demand are never striped with MPK + // protection keys, so this slot has no key to preserve. let mut slot = - MemoryImageSlot::create(mmap_base, byte_size, alloc.byte_capacity()); + MemoryImageSlot::create(mmap_base, byte_size, alloc.byte_capacity(), None); slot.instantiate(alloc.byte_size(), Some(image), ty, memory_tunables)?; Some(slot) } else { diff --git a/crates/wasmtime/src/runtime/vm/mpk/disabled.rs b/crates/wasmtime/src/runtime/vm/mpk/disabled.rs index 6ebd23823738..365dce91d4d2 100644 --- a/crates/wasmtime/src/runtime/vm/mpk/disabled.rs +++ b/crates/wasmtime/src/runtime/vm/mpk/disabled.rs @@ -1,7 +1,7 @@ //! Noop implementations of MPK primitives for environments that do not support //! the feature. -#[cfg(feature = "pooling-allocator")] +#[cfg(any(feature = "pooling-allocator", has_virtual_memory))] use crate::prelude::*; #[cfg(feature = "pooling-allocator")] @@ -34,6 +34,12 @@ impl ProtectionKey { pub fn as_stripe(&self) -> usize { match *self {} } + // Note: gated on `has_virtual_memory` rather than `pooling-allocator` + // because this is called from `cow.rs`, which is not pooling-specific. + #[cfg(has_virtual_memory)] + pub unsafe fn reprotect(&self, _: usize, _: usize, _: bool) -> Result<()> { + match *self {} + } } #[derive(Clone, Copy, Debug)] diff --git a/crates/wasmtime/src/runtime/vm/mpk/enabled.rs b/crates/wasmtime/src/runtime/vm/mpk/enabled.rs index 14ddba4d628f..bef2fb3b1fa1 100644 --- a/crates/wasmtime/src/runtime/vm/mpk/enabled.rs +++ b/crates/wasmtime/src/runtime/vm/mpk/enabled.rs @@ -108,6 +108,35 @@ impl ProtectionKey { pub fn as_stripe(&self) -> usize { self.stripe as usize } + + /// Re-apply this [`ProtectionKey`] to a region that has just been re-mapped. + /// + /// A fresh `mmap` over a region discards that region's protection key, + /// leaving it associated with the default key 0 which is always accessible. + /// Any code that maps over pkey-protected memory must therefore call this + /// afterwards to restore the key, otherwise the memory becomes readable and + /// writable from any stripe. + /// + /// Note that `mprotect` (unlike `mmap`) preserves the existing key, so only + /// `mmap` call sites need this. + /// + /// # Safety + /// + /// `addr` must be page-aligned and `addr..addr + len` must describe a mapped + /// region owned by the caller. `readwrite` must match the page protections + /// the region was just mapped with, since this overwrites them. + pub unsafe fn reprotect(&self, addr: usize, len: usize, readwrite: bool) -> Result<()> { + let prot = if readwrite { + sys::PROT_READ | sys::PROT_WRITE + } else { + sys::PROT_NONE + }; + sys::pkey_mprotect(addr, len, prot, self.id).with_context(|| { + format!( + "failed to restore pkey on region (addr = {addr:#x}, len = {len}, prot = {prot:#b})" + ) + }) + } } /// A bit field indicating which protection keys should be allowed and disabled. diff --git a/crates/wasmtime/src/runtime/vm/mpk/sys.rs b/crates/wasmtime/src/runtime/vm/mpk/sys.rs index fffe53b85dac..f8f1c2776273 100644 --- a/crates/wasmtime/src/runtime/vm/mpk/sys.rs +++ b/crates/wasmtime/src/runtime/vm/mpk/sys.rs @@ -18,6 +18,14 @@ use std::io::Error; /// to start as `PROT_NONE`. pub const PROT_NONE: u32 = libc::PROT_NONE as u32; // == 0b0000; +/// Protection mask allowing reads of pkey-protected memory (see `prot` in +/// [`pkey_mprotect`]). +pub const PROT_READ: u32 = libc::PROT_READ as u32; // == 0b0001; + +/// Protection mask allowing writes of pkey-protected memory (see `prot` in +/// [`pkey_mprotect`]). +pub const PROT_WRITE: u32 = libc::PROT_WRITE as u32; // == 0b0010; + /// Allocate a new protection key in the Linux kernel ([docs]); returns the /// key ID. /// diff --git a/crates/wit-bindgen/src/lib.rs b/crates/wit-bindgen/src/lib.rs index 5154f4491c5f..9b16226f2151 100644 --- a/crates/wit-bindgen/src/lib.rs +++ b/crates/wit-bindgen/src/lib.rs @@ -502,7 +502,7 @@ impl Wasmtime { let key_name = resolve.name_world_key(&key); generator.generate_add_to_linker(id, &key_name); let body = String::from(mem::take(&mut generator.src)); - let interface_name = resolve.interfaces[id].name.as_ref().unwrap(); + let interface_name = to_rust_ident(resolve.interfaces[id].name.as_ref().unwrap()); let body = format!("pub mod {interface_name} {{\n{body}\n}}"); let path = self.generate_interface_name(resolve, id, &key, InterfaceKind::Named); self.named_import_modules diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 66d8309aa9e7..0fa01ef0230a 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -17,6 +17,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-assembler-x64]] version = "0.132.0" audited_as = "0.130.1" @@ -33,6 +37,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-assembler-x64]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-assembler-x64-meta]] version = "0.132.0" audited_as = "0.130.1" @@ -49,6 +57,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-assembler-x64-meta]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-bforest]] version = "0.132.0" audited_as = "0.130.1" @@ -65,6 +77,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-bforest]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-bitset]] version = "0.132.0" audited_as = "0.130.1" @@ -81,6 +97,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-bitset]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-codegen]] version = "0.132.0" audited_as = "0.130.1" @@ -97,6 +117,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-codegen]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-codegen-meta]] version = "0.132.0" audited_as = "0.130.1" @@ -113,6 +137,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-codegen-meta]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-codegen-shared]] version = "0.132.0" audited_as = "0.130.1" @@ -129,6 +157,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-codegen-shared]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-control]] version = "0.132.0" audited_as = "0.130.1" @@ -145,6 +177,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-control]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-entity]] version = "0.132.0" audited_as = "0.130.1" @@ -161,6 +197,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-entity]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-frontend]] version = "0.132.0" audited_as = "0.130.1" @@ -177,6 +217,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-frontend]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-interpreter]] version = "0.132.0" audited_as = "0.130.1" @@ -193,6 +237,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-interpreter]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-isle]] version = "0.132.0" audited_as = "0.130.1" @@ -209,6 +257,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-isle]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-jit]] version = "0.132.0" audited_as = "0.130.1" @@ -225,6 +277,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-jit]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-module]] version = "0.132.0" audited_as = "0.130.1" @@ -241,6 +297,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-module]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-native]] version = "0.132.0" audited_as = "0.130.1" @@ -257,6 +317,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-native]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-object]] version = "0.132.0" audited_as = "0.130.1" @@ -273,6 +337,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-object]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-reader]] version = "0.132.0" audited_as = "0.130.1" @@ -289,6 +357,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-reader]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-serde]] version = "0.132.0" audited_as = "0.130.1" @@ -305,6 +377,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-serde]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.cranelift-srcgen]] version = "0.132.0" audited_as = "0.130.1" @@ -321,6 +397,10 @@ audited_as = "0.132.0" version = "0.135.0" audited_as = "0.133.1" +[[unpublished.cranelift-srcgen]] +version = "0.135.1" +audited_as = "0.135.0" + [[unpublished.pulley-interpreter]] version = "45.0.0" audited_as = "43.0.1" @@ -337,6 +417,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.pulley-interpreter]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.pulley-macros]] version = "45.0.0" audited_as = "43.0.1" @@ -353,6 +437,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.pulley-macros]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasi-common]] version = "45.0.0" audited_as = "43.0.1" @@ -377,6 +465,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-cli]] version = "45.0.0" audited_as = "43.0.1" @@ -393,6 +485,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-cli]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-cli-flags]] version = "45.0.0" audited_as = "43.0.1" @@ -409,6 +505,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-cli-flags]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-environ]] version = "45.0.0" audited_as = "43.0.1" @@ -425,6 +525,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-environ]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-c-api-macros]] version = "45.0.0" audited_as = "43.0.1" @@ -441,6 +545,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-c-api-macros]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-cache]] version = "45.0.0" audited_as = "43.0.1" @@ -457,6 +565,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-cache]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-component-macro]] version = "45.0.0" audited_as = "43.0.1" @@ -473,6 +585,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-component-macro]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-component-util]] version = "45.0.0" audited_as = "43.0.1" @@ -489,6 +605,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-component-util]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-core]] version = "45.0.0" audited_as = "43.0.1" @@ -505,6 +625,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-core]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-cranelift]] version = "45.0.0" audited_as = "43.0.1" @@ -521,6 +645,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-cranelift]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-debugger]] version = "45.0.0" audited_as = "43.0.1" @@ -537,6 +665,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-debugger]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-explorer]] version = "45.0.0" audited_as = "43.0.1" @@ -553,6 +685,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-explorer]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-fiber]] version = "45.0.0" audited_as = "43.0.1" @@ -569,6 +705,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-fiber]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-gdbstub-component-artifact]] version = "45.0.0" audited_as = "44.0.0" @@ -585,6 +725,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-gdbstub-component-artifact]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-jit-debug]] version = "45.0.0" audited_as = "43.0.1" @@ -601,6 +745,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-jit-debug]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-jit-icache-coherence]] version = "45.0.0" audited_as = "43.0.1" @@ -617,6 +765,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-jit-icache-coherence]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-unwinder]] version = "45.0.0" audited_as = "43.0.1" @@ -633,6 +785,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-unwinder]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-versioned-export-macros]] version = "45.0.0" audited_as = "43.0.1" @@ -649,6 +805,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-versioned-export-macros]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-winch]] version = "45.0.0" audited_as = "43.0.1" @@ -665,6 +825,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-winch]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-wit-bindgen]] version = "45.0.0" audited_as = "43.0.1" @@ -681,6 +845,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-wit-bindgen]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-internal-wmemcheck]] version = "45.0.0" audited_as = "43.0.1" @@ -697,6 +865,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-internal-wmemcheck]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wasi]] version = "45.0.0" audited_as = "43.0.1" @@ -713,6 +885,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wasi]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wasi-config]] version = "45.0.0" audited_as = "43.0.1" @@ -729,6 +905,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wasi-config]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wasi-http]] version = "45.0.0" audited_as = "43.0.1" @@ -745,6 +925,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wasi-http]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wasi-io]] version = "45.0.0" audited_as = "43.0.1" @@ -761,6 +945,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wasi-io]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wasi-keyvalue]] version = "45.0.0" audited_as = "43.0.1" @@ -777,6 +965,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wasi-keyvalue]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wasi-nn]] version = "45.0.0" audited_as = "43.0.1" @@ -793,6 +985,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wasi-nn]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wasi-threads]] version = "45.0.0" audited_as = "43.0.1" @@ -817,6 +1013,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wasi-tls]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wast]] version = "45.0.0" audited_as = "43.0.1" @@ -833,6 +1033,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wast]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wasmtime-wizer]] version = "45.0.0" audited_as = "43.0.1" @@ -849,6 +1053,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wasmtime-wizer]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wiggle]] version = "45.0.0" audited_as = "43.0.1" @@ -865,6 +1073,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wiggle]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wiggle-generate]] version = "45.0.0" audited_as = "43.0.1" @@ -881,6 +1093,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wiggle-generate]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wiggle-macro]] version = "45.0.0" audited_as = "43.0.1" @@ -897,6 +1113,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.wiggle-macro]] +version = "48.0.1" +audited_as = "48.0.0" + [[unpublished.wiggle-test]] version = "0.0.0" audited_as = "0.1.0" @@ -917,6 +1137,10 @@ audited_as = "45.0.0" version = "48.0.0" audited_as = "46.0.1" +[[unpublished.winch-codegen]] +version = "48.0.1" +audited_as = "48.0.0" + [[publisher.aho-corasick]] version = "1.0.2" when = "2023-06-04" @@ -1029,22 +1253,22 @@ user-login = "alexcrichton" user-name = "Alex Crichton" [[publisher.cap-fs-ext]] -version = "4.0.2" -when = "2026-02-15" +version = "4.0.3" +when = "2026-08-20" user-id = 6825 user-login = "sunfishcode" user-name = "Dan Gohman" [[publisher.cap-primitives]] -version = "4.0.2" -when = "2026-02-15" +version = "4.0.3" +when = "2026-08-20" user-id = 6825 user-login = "sunfishcode" user-name = "Dan Gohman" [[publisher.cap-std]] -version = "4.0.2" -when = "2026-02-15" +version = "4.0.3" +when = "2026-08-20" user-id = 6825 user-login = "sunfishcode" user-name = "Dan Gohman" @@ -1113,103 +1337,103 @@ user-login = "jrmuizel" user-name = "Jeff Muizelaar" [[publisher.cranelift]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-assembler-x64]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-assembler-x64-meta]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-bforest]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-bitset]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-codegen]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-codegen-meta]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-codegen-shared]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-control]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-entity]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-frontend]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-interpreter]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-isle]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-jit]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-module]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-native]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-object]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-reader]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-serde]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.cranelift-srcgen]] -version = "0.133.1" -when = "2026-06-24" +version = "0.135.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.derive_arbitrary]] @@ -1555,13 +1779,13 @@ user-login = "dtolnay" user-name = "David Tolnay" [[publisher.pulley-interpreter]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.pulley-macros]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.quote]] @@ -1969,153 +2193,153 @@ when = "2026-07-20" trusted-publisher = "github:bytecodealliance/wasm-tools" [[publisher.wasmtime]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-cli]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-cli-flags]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-environ]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-c-api-macros]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-cache]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-component-macro]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-component-util]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-core]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-cranelift]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-debugger]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-explorer]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-fiber]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-gdbstub-component-artifact]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-jit-debug]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-jit-icache-coherence]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-unwinder]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-versioned-export-macros]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-winch]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-wit-bindgen]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-internal-wmemcheck]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wasi]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wasi-config]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wasi-http]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wasi-io]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wasi-keyvalue]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wasi-nn]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wasi-tls]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wast]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wasmtime-wizer]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wast]] @@ -2129,18 +2353,18 @@ when = "2026-07-20" trusted-publisher = "github:bytecodealliance/wasm-tools" [[publisher.wiggle]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wiggle-generate]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wiggle-macro]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.wiggle-test]] @@ -2158,8 +2382,8 @@ user-login = "BurntSushi" user-name = "Andrew Gallant" [[publisher.winch-codegen]] -version = "46.0.1" -when = "2026-06-24" +version = "48.0.0" +when = "2026-08-20" trusted-publisher = "github:bytecodealliance/wasmtime" [[publisher.windows]] diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs index fa3fee6c7429..9c2b8e9b208e 100644 --- a/tests/all/cli_tests.rs +++ b/tests/all/cli_tests.rs @@ -1205,6 +1205,71 @@ mod test_programs { Ok(()) } + #[test] + fn p2_cli_stdout_epipe() -> Result<()> { + let mut child = get_wasmtime_command()? + .args(&["run", "-Wcomponent-model", P2_CLI_STDOUT_EPIPE_COMPONENT]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .stdin(Stdio::null()) + .spawn()?; + + // Read a small amount from stdout then drop it to close the pipe, + // which should cause the guest to receive StreamError::Closed (EPIPE). + let mut stdout = child.stdout.take().unwrap(); + let mut buf = [0u8; 64]; + let _ = stdout.read(&mut buf)?; + drop(stdout); + + let output = child.wait_with_output()?; + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "guest should exit successfully after receiving Closed, stderr: {stderr}" + ); + assert!( + stderr.contains("got expected StreamError::Closed"), + "guest should have reported StreamError::Closed, stderr: {stderr}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn p2_cli_stdin_eisdir() -> Result<()> { + let dir = tempfile::tempdir()?; + // Open the directory and transfer the fd to Stdio for use as stdin. + let dir_file = std::fs::File::open(dir.path())?; + let stdin_stdio: Stdio = dir_file.into(); + + let child = get_wasmtime_command()? + .args(&["run", "-Wcomponent-model", P2_CLI_STDIN_EISDIR_COMPONENT]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .stdin(stdin_stdio) + .spawn()?; + + let output = child.wait_with_output()?; + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "guest should exit successfully after receiving IsDirectory, stderr: {stderr}" + ); + assert!( + stderr.contains("got expected ErrorCode::IsDirectory"), + "guest should have reported ErrorCode::IsDirectory, stderr: {stderr}" + ); + Ok(()) + } + + // EISDIR is a Unix-specific concept; on Windows opening a directory for + // reading behaves differently, so this test only runs on Unix. + #[cfg(not(unix))] + #[test] + fn p2_cli_stdin_eisdir() -> Result<()> { + Ok(()) + } + #[test] fn p2_cli_env() -> Result<()> { run_wasmtime(&[ @@ -3651,3 +3716,21 @@ fn non_utf8_raises_error() -> Result<()> { } Ok(()) } + +#[test] +fn compile_empty_component_with_debug_info() -> Result<()> { + // A component with no core modules reached simulated-DWARF generation with + // nothing to describe, which used to panic instead of compiling. + let td = TempDir::new()?; + let cwasm = td.path().join("empty-component.cwasm"); + let stdout = run_wasmtime(&[ + "compile", + "-D", + "debug-info=y", + "tests/all/cli_tests/empty_component.wat", + "-o", + cwasm.to_str().unwrap(), + ])?; + assert_eq!(stdout, ""); + Ok(()) +} diff --git a/tests/all/cli_tests/empty_component.wat b/tests/all/cli_tests/empty_component.wat new file mode 100644 index 000000000000..e5627d1d0fc7 --- /dev/null +++ b/tests/all/cli_tests/empty_component.wat @@ -0,0 +1 @@ +(component) diff --git a/tests/all/epoch_interruption.rs b/tests/all/epoch_interruption.rs index 6a1c1ff7bb92..1d2b4664f3dc 100644 --- a/tests/all/epoch_interruption.rs +++ b/tests/all/epoch_interruption.rs @@ -496,3 +496,130 @@ async fn drop_future_on_epoch_yield(config: &mut Config) -> Result<()> { assert_eq!(true, alive_flag.load(Ordering::Acquire)); Ok(()) } + +#[test] +fn memory_grow_in_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + config.memory_reservation(0); + config.memory_reservation_for_growth(0); + config.memory_guard_size(0); + config.memory_may_move(true); + config.memory_init_cow(false); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (memory (export "mem") 1) + (func (export "go") + (memory.fill (i32.const 0) (i32.const 0x41) (i32.const 65536)))) + "#, + )?; + + let mut store: Store> = Store::new(&engine, None); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(move |mut cx| { + if let Some(mem) = *cx.data() { + mem.grow(&mut cx, 5)?; + } + Ok(UpdateDeadline::Continue(0)) + }); + + let instance = Instance::new(&mut store, &module, &[])?; + let mem = instance.get_memory(&mut store, "mem").unwrap(); + *store.data_mut() = Some(mem); + engine.increment_epoch(); + + instance + .get_typed_func::<(), ()>(&mut store, "go")? + .call(&mut store, ())?; + + let data = mem.data(&store); + assert_eq!(data[0], 0x41); + assert_eq!(data[65535], 0x41); + Ok(()) +} + +#[test] +fn table_grow_in_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (table $t (export "t") 1 funcref) + (func (export "go") + (table.fill $t (i32.const 0) (ref.null func) (i32.const 1)))) + "#, + )?; + + let mut store: Store> = Store::new(&engine, None); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(move |mut cx| { + if let Some(t) = *cx.data() { + t.grow(&mut cx, 5, Ref::Func(None))?; + } + Ok(UpdateDeadline::Continue(0)) + }); + + let instance = Instance::new(&mut store, &module, &[])?; + let t = instance.get_table(&mut store, "t").unwrap(); + *store.data_mut() = Some(t); + engine.increment_epoch(); + + instance + .get_typed_func::<(), ()>(&mut store, "go")? + .call(&mut store, ())?; + Ok(()) +} + +#[test] +fn gc_during_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (global $sink (mut (ref null $arr)) (ref.null $arr)) + (func $mk (param $n i32) (result (ref $arr)) + (array.new_default $arr (local.get $n))) + (func (export "run") (param $n i32) (result i32) + (local $i i32) + (block $done + (loop $l + (br_if $done (i32.ge_u (local.get $i) (i32.const 40))) + (array.fill $arr (call $mk (local.get $n)) (i32.const 0) + (struct.new $box (i32.const 7)) (local.get $n)) + (global.set $sink (call $mk (i32.const 8))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l) + ) + ) + (i32.mul (local.get $n) (i32.const 7)))) + "#, + )?; + + let mut store = Store::new(&engine, ()); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(|mut caller| { + caller.gc(None)?; + Ok(UpdateDeadline::Continue(0)) + }); + engine.increment_epoch(); + + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::(&mut store, "run")?; + let n = 200; + for i in 0..5 { + let got = run.call(&mut store, n)?; + assert_eq!(got, 7 * n, "iteration {i} read back {got}"); + } + Ok(()) +} diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs index 445a6c1cbd4b..1192fdc46d8c 100644 --- a/tests/all/fuel.rs +++ b/tests/all/fuel.rs @@ -966,6 +966,89 @@ fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> { // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2. let error = grow.call(&mut store, i64::MAX).unwrap_err(); assert_eq!(error.downcast::().unwrap(), Trap::OutOfFuel); + Ok(()) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn huge_table64_grow_cannot_mint_fuel() -> Result<()> { + huge_table64_grow_cannot_mint_fuel_impl( + r#" + (module + (table $t i64 0 0x10000 (ref null func)) + (func (export "run") (param $delta i64) + (loop $l + (drop (table.grow $t (ref.null func) (local.get $delta))) + (br $l)))) + "#, + ) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn huge_table64_grow_cannot_mint_fuel_const() -> Result<()> { + huge_table64_grow_cannot_mint_fuel_impl( + r#" + (module + (table $t i64 0 0x10000 (ref null func)) + (func (export "run") (param $delta i64) + (loop $l + (drop (table.grow $t (ref.null func) (i64.const -500))) + (br $l)))) + "#, + ) +} + +fn huge_table64_grow_cannot_mint_fuel_impl(wat: &str) -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let module = Module::new(&engine, wat)?; + + let mut store = Store::new(&engine, ()); + store.set_fuel(100_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::(&mut store, "run")?; + + let trap = run.call(&mut store, -500).unwrap_err().downcast::()?; + assert_eq!(trap, Trap::OutOfFuel); + assert_eq!(store.get_fuel()?, 0); + Ok(()) +} +#[test] +#[cfg_attr(miri, ignore)] +fn fuel_around_table_grow() -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $ft (func)) + (func $f (type $ft)) + (table $t 1 10000000 (ref $ft) (ref.func $f)) + (func (export "grow") (result i32) + (table.grow $t (ref.func $f) (i32.const 9999999))) + (func (export "call") (param i32) + (call_indirect $t (type $ft) (local.get 0)))) + "#, + )?; + + let mut store = Store::new(&engine, ()); + store.set_fuel(2)?; + let instance = Instance::new(&mut store, &module, &[])?; + let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?; + let trap = grow.call(&mut store, ()).unwrap_err().downcast::()?; + assert_eq!(trap, Trap::OutOfFuel); + + store.set_fuel(u64::MAX)?; + let call = instance.get_typed_func::(&mut store, "call")?; + let trap = call + .call(&mut store, 9999999) + .unwrap_err() + .downcast::()?; + assert_eq!(trap, Trap::TableOutOfBounds); Ok(()) } diff --git a/tests/all/gc.rs b/tests/all/gc.rs index 3058855e3335..79a049da8afd 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -3788,3 +3788,107 @@ fn initial_size_larger_than_reservation() -> Result<()> { Ok(()) } + +#[test] +#[cfg_attr(miri, ignore)] +fn array_fill_i64_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $arr (array (mut i64))) + (type $box (struct (field i32))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $i i32) (local $s i32) + (local.set $a (array.new_default $arr (local.get $n))) + ;; Keep the collector busy so it has something to move. + (drop (struct.new $box (i32.const 1))) + (array.fill $arr (local.get $a) (i32.const 0) (i64.const 7) (local.get $n)) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (i32.wrap_i64 (array.get $arr (local.get $a) (local.get $i))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn array_new_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $i i32) (local $s i32) + (local.set $a (array.new $arr (struct.new $box (i32.const 7)) (local.get $n))) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (struct.get $box 0 (ref.as_non_null + (array.get $arr (local.get $a) (local.get $i)))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn array_copy_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $b (ref null $arr)) + (local $i i32) (local $s i32) + (local.set $a (array.new_default $arr (local.get $n))) + (local.set $b (array.new_default $arr (local.get $n))) + (array.fill $arr (local.get $b) (i32.const 0) + (struct.new $box (i32.const 7)) (local.get $n)) + (array.copy $arr $arr (local.get $a) (i32.const 0) + (local.get $b) (i32.const 0) (local.get $n)) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (struct.get $box 0 (ref.as_non_null + (array.get $arr (local.get $a) (local.get $i)))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +fn gc_during_epoch(wat: &str) -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new(&engine, wat)?; + + let mut store = Store::new(&engine, ()); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(|mut caller| { + caller.gc(None)?; + Ok(UpdateDeadline::Continue(0)) + }); + engine.increment_epoch(); + + let instance = Instance::new(&mut store, &module, &[])?; + let f = instance.get_typed_func::(&mut store, "run")?; + + let n = 100; + for i in 0..5 { + match f.call(&mut store, n) { + Ok(got) => assert_eq!(got, 7 * n, "iteration {i} read back {got}"), + Err(e) => panic!("iteration {i} failed: {e:?}"), + } + } + Ok(()) +} diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs index fc19983934ca..e556f233c269 100644 --- a/tests/all/pooling_allocator.rs +++ b/tests/all/pooling_allocator.rs @@ -1556,3 +1556,96 @@ fn purge_module_with_mpk() -> Result<()> { Ok(()) } + +/// Regression test for both #7942 and #13982: mapping a copy-on-write memory +/// image into a slot must not drop the slot's MPK protection key. +/// +/// A fresh `mmap` associates the pages it replaces with the default protection +/// key 0, which every stripe is allowed to access. If the key is not +/// re-applied afterwards, an instance in one stripe can read and write the +/// linear memory of an instance in another stripe. +#[test] +#[cfg_attr(miri, ignore)] +fn mpk_protects_memory_images() -> Result<()> { + if !wasmtime::PoolingAllocationConfig::are_memory_protection_keys_available() { + println!("skipping test; mpk is not supported"); + return Ok(()); + } + + let mut pool = wasmtime::PoolingAllocationConfig::new(); + pool.memory_protection_keys(Enabled::Yes) + .max_memory_protection_keys(2) + .max_memory_size(1 << 20) + .total_memories(4) + .total_tables(4) + .total_core_instances(4); + let mut config = Config::new(); + config.allocation_strategy(InstanceAllocationStrategy::Pooling(pool)); + let engine = Engine::new(&config)?; + + // The victim has a `(data ...)` segment, so it is instantiated from a + // copy-on-write memory image; that is the mapping which used to clobber + // the protection key. + let victim = Module::new( + &engine, + r#"(module (memory (export "m") 1) (data (i32.const 0) "SECRET"))"#, + )?; + let attacker = Module::new( + &engine, + r#"(module + (memory (export "m") 1) + (func (export "load") (param i32) (result i32) local.get 0 i32.load) + (func (export "store") (param i32 i32) local.get 0 local.get 1 i32.store))"#, + )?; + + let mut attacker_store = Store::new(&engine, ()); + let mut victim_store = Store::new(&engine, ()); + let attacker_instance = Instance::new(&mut attacker_store, &attacker, &[])?; + let victim_instance = Instance::new(&mut victim_store, &victim, &[])?; + + let attacker_mem = attacker_instance + .get_memory(&mut attacker_store, "m") + .unwrap(); + let victim_mem = victim_instance.get_memory(&mut victim_store, "m").unwrap(); + + // Only meaningful if the two instances landed in different stripes and the + // victim is within reach of a 32-bit wasm address. + let attacker_base = attacker_mem.data_ptr(&attacker_store) as usize; + let victim_base = victim_mem.data_ptr(&victim_store) as usize; + let offset = match victim_base + .checked_sub(attacker_base) + .and_then(|offset| u32::try_from(offset).ok()) + { + // Wasm addresses are unsigned, so this is a plain bit-cast. + Some(offset) => offset as i32, + None => { + println!("skipping test; victim memory is not addressable by the attacker"); + return Ok(()); + } + }; + + let load = attacker_instance.get_typed_func::(&mut attacker_store, "load")?; + let store = attacker_instance.get_typed_func::<(i32, i32), ()>(&mut attacker_store, "store")?; + + // The attacker can still use its own memory... + store.call(&mut attacker_store, (0, 0x12345678))?; + assert_eq!(load.call(&mut attacker_store, 0)?, 0x12345678); + + // ...and the victim's image was still applied correctly... + assert_eq!(&victim_mem.data(&victim_store)[..6], b"SECRET"); + + // ...but it must not be able to touch the victim's memory. + assert!( + load.call(&mut attacker_store, offset).is_err(), + "attacker read across an MPK stripe boundary" + ); + assert!( + store + .call(&mut attacker_store, (offset, 0x41414141)) + .is_err(), + "attacker wrote across an MPK stripe boundary" + ); + assert_eq!(&victim_mem.data(&victim_store)[..6], b"SECRET"); + + Ok(()) +} diff --git a/tests/disas/component-model/direct-adapter-calls-inlining.wat b/tests/disas/component-model/direct-adapter-calls-inlining.wat index 04435c56b5d6..d7900b80b5ba 100644 --- a/tests/disas/component-model/direct-adapter-calls-inlining.wat +++ b/tests/disas/component-model/direct-adapter-calls-inlining.wat @@ -104,7 +104,6 @@ ;; block9: ;; v11 = load.i64 notrap aligned readonly can_move region3 v3+112 ;; v12 = load.i32 notrap aligned region4 v11 -;; store notrap aligned region4 v12, v11 ;; jump block13 ;; ;; block13: diff --git a/tests/disas/component-model/direct-adapter-calls-x64.wat b/tests/disas/component-model/direct-adapter-calls-x64.wat index cfa4fead7efa..8831a79b6fc8 100644 --- a/tests/disas/component-model/direct-adapter-calls-x64.wat +++ b/tests/disas/component-model/direct-adapter-calls-x64.wat @@ -87,7 +87,7 @@ ;; movq 0x18(%r10), %r10 ;; addq $0x60, %r10 ;; cmpq %rsp, %r10 -;; ja 0x147 +;; ja 0x13e ;; 79: subq $0x50, %rsp ;; movq %rbx, 0x20(%rsp) ;; movq %r12, 0x28(%rsp) @@ -97,10 +97,10 @@ ;; movq %rdi, (%rsp) ;; movq (%rsp), %rdi ;; movq 0x88(%rdi), %rcx -;; movl (%rcx), %eax +;; movl (%rcx), %esi ;; movq %rcx, 0x10(%rsp) -;; testl %eax, %eax -;; movq %rax, 8(%rsp) +;; testl %esi, %esi +;; movq %rsi, 8(%rsp) ;; jne 0xd5 ;; b9: movq (%rsp), %rdi ;; movq 0x58(%rdi), %rax @@ -109,24 +109,19 @@ ;; movq (%rsp), %rsi ;; callq *%rax ;; ├─╼ exception frame offset: SP = FP - 0x50 -;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x132 -;; jmp 0x130 -;; d5: movq (%rsp), %rsi -;; movq 0x70(%rsi), %rax -;; movl (%rax), %ecx -;; movl %ecx, (%rax) -;; movq 0x48(%rsi), %rdi +;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x121 +;; jmp 0x11f +;; d5: movq (%rsp), %rcx +;; movq 0x70(%rcx), %rax +;; movl (%rax), %eax +;; movq 0x48(%rcx), %rdi +;; movq (%rsp), %rsi ;; callq 0 ;; ├─╼ exception frame offset: SP = FP - 0x50 -;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0xef -;; jmp 0xf7 -;; ef: movq %rax, %rdx -;; jmp 0x132 -;; f7: movq %rax, %rdx -;; movq 8(%rsp), %rcx -;; movq 0x10(%rsp), %rax -;; movl %ecx, (%rax) -;; movq %rdx, %rax +;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x121 +;; movq 0x10(%rsp), %rcx +;; movq 8(%rsp), %rsi +;; movl %esi, (%rcx) ;; movq 0x20(%rsp), %rbx ;; movq 0x28(%rsp), %r12 ;; movq 0x30(%rsp), %r13 @@ -136,12 +131,14 @@ ;; movq %rbp, %rsp ;; popq %rbp ;; retq -;; 12b: jmp 0x132 -;; 130: ud2 -;; 132: movq (%rsp), %rsi -;; 136: movq 0x58(%rsi), %rcx -;; 13a: movq 0x68(%rsi), %rdi -;; 13e: movl $0x31, %edx -;; 143: callq *%rcx -;; 145: ud2 -;; 147: ud2 +;; 11a: jmp 0x121 +;; 11f: ud2 +;; 121: movq (%rsp), %rcx +;; 125: movq 0x58(%rcx), %rcx +;; 129: movq (%rsp), %rax +;; 12d: movq 0x68(%rax), %rdi +;; 131: movl $0x31, %edx +;; 136: movq (%rsp), %rsi +;; 13a: callq *%rcx +;; 13c: ud2 +;; 13e: ud2 diff --git a/tests/disas/component-model/direct-adapter-calls.wat b/tests/disas/component-model/direct-adapter-calls.wat index e3ce501b14d6..870554c73541 100644 --- a/tests/disas/component-model/direct-adapter-calls.wat +++ b/tests/disas/component-model/direct-adapter-calls.wat @@ -133,7 +133,6 @@ ;; block7: ;; @008e v11 = load.i64 notrap aligned readonly can_move region2 v0+112 ;; @008e v12 = load.i32 notrap aligned region3 v11 -;; @009a store notrap aligned region3 v12, v11 ;; @009c v16 = load.i64 notrap aligned readonly can_move region4 v0+72 ;; @009c try_call fn0(v16, v0, v2), sig1, block10(ret0), [ context v0, default: block6(exn0) ] ;; diff --git a/tests/disas/component-model/sync-adapter-calls.wat b/tests/disas/component-model/sync-adapter-calls.wat index 36536000f4cc..c8836867ab6c 100644 --- a/tests/disas/component-model/sync-adapter-calls.wat +++ b/tests/disas/component-model/sync-adapter-calls.wat @@ -149,7 +149,6 @@ ;; store notrap aligned region5 v19, v20+136 ;; v26 = load.i64 notrap aligned readonly can_move region3 v3+176 ;; v27 = load.i32 notrap aligned region4 v26 -;; store notrap aligned region4 v27, v26 ;; jump block17 ;; ;; block17: @@ -269,7 +268,6 @@ ;; @00f0 store notrap aligned region6 v19, v20+136 ;; @00f2 v26 = load.i64 notrap aligned readonly can_move region2 v0+176 ;; @00f2 v27 = load.i32 notrap aligned region3 v26 -;; @00fe store notrap aligned region3 v27, v26 ;; @0100 jump block15 ;; ;; block15: diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.wat index aeab1b17ebd3..8371c0eed235 100644 --- a/tests/disas/gc/array-copy-with-fuel.wat +++ b/tests/disas/gc/array-copy-with-fuel.wat @@ -28,10 +28,10 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32): -;; v181 = stack_addr.i64 ss0 -;; store notrap aligned region6 v2, v181 -;; v182 = stack_addr.i64 ss1 -;; store notrap aligned region7 v4, v182 +;; v162 = stack_addr.i64 ss0 +;; store notrap aligned region6 v2, v162 +;; v163 = stack_addr.i64 ss1 +;; store notrap aligned region7 v4, v163 ;; @0020 v7 = load.i64 notrap aligned readonly can_move region0 v0+8 ;; @0020 v8 = load.i64 notrap aligned region2 v7 ;; @0020 v9 = iconst.i64 1 @@ -41,131 +41,108 @@ ;; @0020 brif v12, block2, block3(v10) ;; ;; block2: -;; v191 = iadd.i64 v8, v9 ; v9 = 1 -;; @0020 store notrap aligned region2 v191, v7 +;; v172 = iadd.i64 v8, v9 ; v9 = 1 +;; @0020 store notrap aligned region2 v172, v7 ;; @0020 v14 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] ;; @0020 v16 = load.i64 notrap aligned region2 v7 ;; @0020 jump block3(v16) ;; -;; block3(v89: i64): -;; v180 = load.i32 notrap aligned region6 v181 -;; @002b trapz v180, user16 -;; @002b v24 = load.i64 notrap aligned readonly can_move region3 v7+32 -;; @002b v22 = uextend.i64 v180 -;; @002b v25 = iadd v24, v22 -;; @002b v26 = iconst.i64 16 -;; @002b v27 = iadd v25, v26 ; v26 = 16 -;; @002b v28 = load.i32 user2 readonly region5 v27 -;; @002b v30 = uextend.i64 v3 -;; @002b v31 = uextend.i64 v6 -;; @002b v34 = iadd v30, v31 -;; @002b v29 = uextend.i64 v28 -;; @002b v35 = icmp ugt v34, v29 -;; @002b trapnz v35, user17 -;; v174 = load.i32 notrap aligned region7 v182 -;; @002b trapz v174, user16 -;; @002b v46 = uextend.i64 v174 -;; @002b v49 = iadd v24, v46 -;; @002b v51 = iadd v49, v26 ; v26 = 16 -;; @002b v52 = load.i32 user2 readonly region5 v51 -;; @002b v54 = uextend.i64 v5 -;; @002b v58 = iadd v54, v31 -;; @002b v53 = uextend.i64 v52 -;; @002b v59 = icmp ugt v58, v53 -;; @002b trapnz v59, user17 -;; @002b v78 = load.i64 notrap aligned region4 v7+40 -;; @002b v40 = iconst.i64 20 -;; @002b v41 = iadd v25, v40 ; v40 = 20 -;; v184 = iconst.i64 2 -;; v185 = ishl v30, v184 ; v184 = 2 -;; @002b v45 = iadd v41, v185 -;; v189 = ishl v31, v184 ; v184 = 2 -;; @002b v80 = uadd_overflow_trap v45, v189, user2 -;; @002b v79 = iadd v24, v78 -;; @002b v81 = icmp ugt v80, v79 -;; @002b trapnz v81, user2 -;; @002b v65 = iadd v49, v40 ; v40 = 20 -;; v187 = ishl v54, v184 ; v184 = 2 -;; @002b v69 = iadd v65, v187 -;; @002b v87 = uadd_overflow_trap v69, v189, user2 -;; @002b v88 = icmp ugt v87, v79 -;; @002b trapnz v88, user2 -;; @002b v90 = iconst.i64 6 -;; @002b v91 = iadd v89, v90 ; v90 = 6 -;; @002b brif.i32 v6, block4, block7(v91) +;; block3(v25: i64): +;; @002b v26 = iconst.i64 6 +;; @002b v27 = iadd v25, v26 ; v26 = 6 +;; @002b v22 = uextend.i64 v6 +;; @002b v28 = iadd v27, v22 +;; v173 = iconst.i64 0 +;; v174 = icmp sge v28, v173 ; v173 = 0 +;; @002b brif v174, block4, block5(v28) ;; ;; block4: -;; v162 = load.i32 notrap aligned region6 v181 -;; v164 = load.i32 notrap aligned region7 v182 -;; @002b v92 = icmp.i64 ult v45, v69 -;; v192 = iadd.i64 v89, v90 ; v90 = 6 -;; @002b v97 = iadd.i64 v45, v189 -;; @002b v98 = iadd.i64 v69, v189 -;; @002b v100 = iadd.i32 v5, v6 -;; @002b v43 = iconst.i64 4 -;; @002b v141 = iconst.i32 1 -;; @002b brif v92, block5(v45, v69, v5, v162, v164, v192), block6(v97, v98, v100, v162, v164, v192) +;; @002b store.i64 notrap aligned region2 v28, v7 +;; @002b v32 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] +;; @002b v34 = load.i64 notrap aligned region2 v7 +;; @002b jump block5(v34) ;; -;; block5(v101: i64, v102: i64, v103: i32, v104: i32, v105: i32, v106: i64): -;; store notrap aligned region6 v104, v181 -;; store notrap aligned region7 v105, v182 -;; v202 = iconst.i64 1 -;; v203 = iadd v106, v202 ; v202 = 1 -;; v204 = iconst.i64 0 -;; v205 = icmp sge v203, v204 ; v204 = 0 -;; @002b brif v205, block8, block9(v203) +;; block5(v139: i64): +;; v161 = load.i32 notrap aligned region6 v162 +;; @002b trapz v161, user16 +;; @002b v37 = load.i64 notrap aligned readonly can_move region3 v7+32 +;; @002b v35 = uextend.i64 v161 +;; @002b v38 = iadd v37, v35 +;; @002b v39 = iconst.i64 16 +;; @002b v40 = iadd v38, v39 ; v39 = 16 +;; @002b v41 = load.i32 user2 readonly region5 v40 +;; @002b v43 = uextend.i64 v3 +;; @002b v47 = iadd v43, v22 +;; @002b v42 = uextend.i64 v41 +;; @002b v48 = icmp ugt v47, v42 +;; @002b trapnz v48, user17 +;; v155 = load.i32 notrap aligned region7 v163 +;; @002b trapz v155, user16 +;; @002b v59 = uextend.i64 v155 +;; @002b v62 = iadd v37, v59 +;; @002b v64 = iadd v62, v39 ; v39 = 16 +;; @002b v65 = load.i32 user2 readonly region5 v64 +;; @002b v67 = uextend.i64 v5 +;; @002b v71 = iadd v67, v22 +;; @002b v66 = uextend.i64 v65 +;; @002b v72 = icmp ugt v71, v66 +;; @002b trapnz v72, user17 +;; @002b v91 = load.i64 notrap aligned region4 v7+40 +;; @002b v53 = iconst.i64 20 +;; @002b v54 = iadd v38, v53 ; v53 = 20 +;; v165 = iconst.i64 2 +;; v166 = ishl v43, v165 ; v165 = 2 +;; @002b v58 = iadd v54, v166 +;; v170 = ishl.i64 v22, v165 ; v165 = 2 +;; @002b v93 = uadd_overflow_trap v58, v170, user2 +;; @002b v92 = iadd v37, v91 +;; @002b v94 = icmp ugt v93, v92 +;; @002b trapnz v94, user2 +;; @002b v78 = iadd v62, v53 ; v53 = 20 +;; v168 = ishl v67, v165 ; v165 = 2 +;; @002b v82 = iadd v78, v168 +;; @002b v100 = uadd_overflow_trap v82, v170, user2 +;; @002b v101 = icmp ugt v100, v92 +;; @002b trapnz v101, user2 +;; @002b brif.i32 v6, block6, block9 ;; -;; block6(v123: i64, v124: i64, v125: i32, v126: i32, v127: i32, v128: i64): -;; store notrap aligned region7 v126, v182 -;; store notrap aligned region6 v127, v181 -;; v193 = iconst.i64 1 -;; v194 = iadd v128, v193 ; v193 = 1 -;; v195 = iconst.i64 0 -;; v196 = icmp sge v194, v195 ; v195 = 0 -;; @002b brif v196, block10, block11(v194) +;; block6: +;; v143 = load.i32 notrap aligned region6 v162 +;; v145 = load.i32 notrap aligned region7 v163 +;; @002b v102 = icmp.i64 ult v58, v82 +;; @002b v107 = iadd.i64 v58, v170 +;; @002b v108 = iadd.i64 v82, v170 +;; @002b v110 = iadd.i32 v5, v6 +;; @002b v56 = iconst.i64 4 +;; @002b v133 = iconst.i32 1 +;; @002b brif v102, block7(v58, v82, v5), block8(v107, v108, v110) ;; -;; block7(v148: i64): -;; @002f jump block1 -;; -;; block8: -;; @002b store.i64 notrap aligned region2 v203, v7 -;; @002b v112 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] -;; @002b v114 = load.i64 notrap aligned region2 v7 -;; @002b jump block9(v114) +;; block7(v111: i64, v112: i64, v113: i32): +;; @002b v116 = load.i32 user2 little region5 v112 +;; @002b store user2 little region5 v116, v111 +;; v180 = iconst.i64 4 +;; v181 = iadd v112, v180 ; v180 = 4 +;; @002b v123 = icmp eq v181, v108 +;; v182 = iadd v111, v180 ; v180 = 4 +;; v183 = iconst.i32 1 +;; v184 = iadd v113, v183 ; v183 = 1 +;; @002b brif v123, block9, block7(v182, v181, v184) ;; -;; block9(v145: i64): -;; @002b v115 = load.i32 user2 little region5 v102 -;; @002b store user2 little region5 v115, v101 -;; v150 = load.i32 notrap aligned region6 v181 -;; v152 = load.i32 notrap aligned region7 v182 -;; v206 = iconst.i64 4 -;; v207 = iadd.i64 v102, v206 ; v206 = 4 -;; @002b v122 = icmp eq v207, v98 -;; v208 = iadd.i64 v101, v206 ; v206 = 4 -;; v209 = iconst.i32 1 -;; v210 = iadd.i32 v103, v209 ; v209 = 1 -;; @002b brif v122, block7(v145), block5(v208, v207, v210, v150, v152, v145) +;; block8(v124: i64, v125: i64, v126: i32): +;; v175 = iconst.i64 4 +;; v176 = isub v125, v175 ; v175 = 4 +;; @002b v135 = load.i32 user2 little region5 v176 +;; v177 = isub v124, v175 ; v175 = 4 +;; @002b store user2 little region5 v135, v177 +;; @002b v136 = icmp eq v176, v82 +;; v178 = iconst.i32 1 +;; v179 = isub v126, v178 ; v178 = 1 +;; @002b brif v136, block9, block8(v177, v176, v179) ;; -;; block10: -;; @002b store.i64 notrap aligned region2 v194, v7 -;; @002b v134 = call fn0(v0), stack_map=[i32 @ ss1+0, i32 @ ss0+0] -;; @002b v136 = load.i64 notrap aligned region2 v7 -;; @002b jump block11(v136) -;; -;; block11(v146: i64): -;; v197 = iconst.i64 4 -;; v198 = isub.i64 v124, v197 ; v197 = 4 -;; @002b v143 = load.i32 user2 little region5 v198 -;; v199 = isub.i64 v123, v197 ; v197 = 4 -;; @002b store user2 little region5 v143, v199 -;; v156 = load.i32 notrap aligned region7 v182 -;; v158 = load.i32 notrap aligned region6 v181 -;; @002b v144 = icmp eq v198, v69 -;; v200 = iconst.i32 1 -;; v201 = isub.i32 v125, v200 ; v200 = 1 -;; @002b brif v144, block7(v146), block6(v199, v198, v201, v156, v158, v146) +;; block9: +;; @002f jump block1 ;; ;; block1: -;; @002f store.i64 notrap aligned region2 v148, v7 +;; @002f store.i64 notrap aligned region2 v139, v7 ;; @002f return ;; } diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.wat index f712972b8774..d6113471a161 100644 --- a/tests/disas/memory-copy-epochs.wat +++ b/tests/disas/memory-copy-epochs.wat @@ -37,101 +37,35 @@ ;; @001e v10 = call fn0(v0) ;; @001e jump block2(v10) ;; -;; block2(v59: i64): -;; @0025 v14 = load.i64 notrap aligned region6 v0+64 -;; @0025 v15 = uextend.i64 v2 -;; @0025 v16 = uextend.i64 v4 -;; @0025 v19 = iadd v15, v16 -;; @0025 v20 = icmp ugt v19, v14 -;; @0025 trapnz v20, heap_oob -;; @0025 v27 = uextend.i64 v3 -;; @0025 v31 = iadd v27, v16 -;; @0025 v32 = icmp ugt v31, v14 -;; @0025 trapnz v32, heap_oob -;; @0025 v21 = load.i64 notrap aligned readonly can_move region5 v0+56 -;; @0025 v37 = iadd v21, v27 -;; @0025 v25 = iadd v21, v15 -;; @0025 v40 = icmp ugt v37, v25 -;; @0025 brif v40, block6, block7 -;; -;; block4(v42: i64, v43: i64, v44: i64, v47: i64): -;; @0025 v46 = load.i64 notrap aligned region3 v5 -;; @0025 v48 = icmp uge v46, v47 -;; @0025 brif v48, block9, block8(v47) -;; -;; block5(v86: i64, v87: i64, v88: i64, v92: i64): -;; @0025 v91 = load.i64 notrap aligned region3 v5 -;; @0025 v94 = icmp uge v91, v92 -;; @0025 brif v94, block17, block16 -;; -;; block6: -;; v108 = iconst.i64 0x0800_0000 -;; v109 = icmp.i64 ugt v16, v108 ; v108 = 0x0800_0000 -;; @0025 brif v109, block4(v25, v37, v16, v59), block5(v25, v37, v16, v59) -;; -;; block9 cold: -;; @0025 v50 = load.i64 notrap aligned region4 v7+8 -;; @0025 v51 = icmp.i64 uge v46, v50 -;; @0025 brif v51, block10, block8(v50) -;; -;; block10 cold: -;; @0025 v52 = call fn0(v0) -;; @0025 jump block8(v52) -;; -;; block8(v60: i64): -;; @0025 call fn1(v0, v42, v43, v108) ; v108 = 0x0800_0000 -;; @0025 v55 = isub.i64 v44, v108 ; v108 = 0x0800_0000 -;; @0025 v56 = icmp ugt v55, v108 ; v108 = 0x0800_0000 -;; @0025 v53 = iadd.i64 v42, v108 ; v108 = 0x0800_0000 -;; @0025 v54 = iadd.i64 v43, v108 ; v108 = 0x0800_0000 -;; @0025 brif v56, block4(v53, v54, v55, v60), block5(v53, v54, v55, v60) -;; -;; block7: -;; @0025 v39 = iconst.i64 0x0800_0000 -;; @0025 v63 = icmp.i64 ugt v16, v39 ; v39 = 0x0800_0000 -;; @0025 v61 = iadd.i64 v25, v16 -;; @0025 v62 = iadd.i64 v37, v16 -;; @0025 brif v63, block11(v61, v62, v16, v59), block12(v61, v62, v16, v59) -;; -;; block11(v64: i64, v65: i64, v66: i64, v71: i64): -;; @0025 v70 = load.i64 notrap aligned region3 v5 -;; @0025 v72 = icmp uge v70, v71 -;; @0025 brif v72, block14, block13(v71) -;; -;; block14 cold: -;; @0025 v74 = load.i64 notrap aligned region4 v7+8 -;; @0025 v75 = icmp.i64 uge v70, v74 -;; @0025 brif v75, block15, block13(v74) -;; -;; block15 cold: -;; @0025 v76 = call fn0(v0) -;; @0025 jump block13(v76) -;; -;; block13(v80: i64): -;; v103 = iconst.i64 0x0800_0000 -;; v104 = isub.i64 v64, v103 ; v103 = 0x0800_0000 -;; v105 = isub.i64 v65, v103 ; v103 = 0x0800_0000 -;; @0025 call fn1(v0, v104, v105, v103) ; v103 = 0x0800_0000 -;; v106 = isub.i64 v66, v103 ; v103 = 0x0800_0000 -;; v107 = icmp ugt v106, v103 ; v103 = 0x0800_0000 -;; @0025 brif v107, block11(v104, v105, v106, v80), block12(v104, v105, v106, v80) -;; -;; block12(v81: i64, v82: i64, v83: i64, v93: i64): -;; @0025 v84 = isub v81, v83 -;; @0025 v85 = isub v82, v83 -;; @0025 jump block5(v84, v85, v83, v93) -;; -;; block17 cold: -;; @0025 v96 = load.i64 notrap aligned region4 v7+8 -;; @0025 v97 = icmp.i64 uge v91, v96 -;; @0025 brif v97, block18, block16 -;; -;; block18 cold: -;; @0025 v98 = call fn0(v0) -;; @0025 jump block16 -;; -;; block16: -;; @0025 call fn1(v0, v86, v87, v88) +;; block2(v16: i64): +;; @0025 v15 = load.i64 notrap aligned region3 v5 +;; @0025 v17 = icmp uge v15, v16 +;; @0025 brif v17, block5, block4 +;; +;; block5 cold: +;; @0025 v19 = load.i64 notrap aligned region4 v7+8 +;; @0025 v20 = icmp.i64 uge v15, v19 +;; @0025 brif v20, block6, block4 +;; +;; block6 cold: +;; @0025 v21 = call fn0(v0) +;; @0025 jump block4 +;; +;; block4: +;; @0025 v22 = load.i64 notrap aligned region6 v0+64 +;; @0025 v23 = uextend.i64 v2 +;; @0025 v24 = uextend.i64 v4 +;; @0025 v27 = iadd v23, v24 +;; @0025 v28 = icmp ugt v27, v22 +;; @0025 trapnz v28, heap_oob +;; @0025 v35 = uextend.i64 v3 +;; @0025 v39 = iadd v35, v24 +;; @0025 v40 = icmp ugt v39, v22 +;; @0025 trapnz v40, heap_oob +;; @0025 v29 = load.i64 notrap aligned readonly can_move region5 v0+56 +;; @0025 v33 = iadd v29, v23 +;; @0025 v45 = iadd v29, v35 +;; @0025 call fn1(v0, v33, v45, v24) ;; @0029 jump block1 ;; ;; block1: diff --git a/tests/disas/memory-copy-fuel-const-len.wat b/tests/disas/memory-copy-fuel-const-len.wat new file mode 100644 index 000000000000..aee7af900c1b --- /dev/null +++ b/tests/disas/memory-copy-fuel-const-len.wat @@ -0,0 +1,184 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! flags = '-Wfuel=100' + +(module + (memory 1) + (func $copy_16 (param i32 i32) + (memory.copy (local.get 0) (local.get 1) (i32.const 16)) + ) + (func $fill_128 (param i32) + (memory.fill (local.get 0) (i32.const 0) (i32.const 128)) + ) + (func $fill_4096 (param i32) + (memory.fill (local.get 0) (i32.const 0) (i32.const 4096)) + ) +) +;; function u0:0(i64 vmctx, i64, i32, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; region5 = 201326592 "DefinedMemory(StaticModuleIndex(0), DefinedMemoryIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; fn0 = colocated u805306368:12 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32, v3: i32): +;; @0023 v4 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0023 v5 = load.i64 notrap aligned region2 v4 +;; @0023 v6 = iconst.i64 1 +;; @0023 v7 = iadd v5, v6 ; v6 = 1 +;; @0023 v8 = iconst.i64 0 +;; @0023 v9 = icmp sge v7, v8 ; v8 = 0 +;; @0023 brif v9, block2, block3(v7) +;; +;; block2: +;; v58 = iadd.i64 v5, v6 ; v6 = 1 +;; @0023 store notrap aligned region2 v58, v4 +;; @0023 v11 = call fn0(v0) +;; @0023 v13 = load.i64 notrap aligned region2 v4 +;; @0023 jump block3(v13) +;; +;; block3(v43: i64): +;; @002a v17 = load.i64 notrap aligned region4 v0+64 +;; @002a v18 = uextend.i64 v2 +;; v47 = iconst.i64 16 +;; @002a v22 = iadd v18, v47 ; v47 = 16 +;; @002a v23 = icmp ugt v22, v17 +;; @002a trapnz v23, heap_oob +;; @002a v30 = uextend.i64 v3 +;; @002a v34 = iadd v30, v47 ; v47 = 16 +;; @002a v35 = icmp ugt v34, v17 +;; @002a trapnz v35, heap_oob +;; @002a v24 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @002a v40 = iadd v24, v30 +;; @002a v42 = load.i8x16 notrap aligned little region5 v40 +;; @002a v28 = iadd v24, v18 +;; @002a store notrap aligned little region5 v42, v28 +;; @002e jump block1 +;; +;; block1: +;; @002e v44 = iconst.i64 20 +;; @002e v45 = iadd.i64 v43, v44 ; v44 = 20 +;; @002e store notrap aligned region2 v45, v4 +;; @002e return +;; } +;; +;; function u0:1(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; fn0 = colocated u805306368:12 sig0 +;; fn1 = colocated u805306368:2 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0030 v3 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0030 v4 = load.i64 notrap aligned region2 v3 +;; @0030 v5 = iconst.i64 1 +;; @0030 v6 = iadd v4, v5 ; v5 = 1 +;; @0030 v7 = iconst.i64 0 +;; @0030 v8 = icmp sge v6, v7 ; v7 = 0 +;; @0030 brif v8, block2, block3(v6) +;; +;; block2: +;; v43 = iadd.i64 v4, v5 ; v5 = 1 +;; @0030 store notrap aligned region2 v43, v3 +;; @0030 v10 = call fn0(v0) +;; @0030 v12 = load.i64 notrap aligned region2 v3 +;; @0030 jump block3(v12) +;; +;; block3(v29: i64): +;; @0038 v16 = load.i64 notrap aligned region4 v0+64 +;; @0038 v17 = uextend.i64 v2 +;; v33 = iconst.i64 128 +;; @0038 v21 = iadd v17, v33 ; v33 = 128 +;; @0038 v22 = icmp ugt v21, v16 +;; @0038 trapnz v22, heap_oob +;; @0038 v23 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0038 v27 = iadd v23, v17 +;; @0033 v14 = iconst.i32 0 +;; @0038 call fn1(v0, v27, v14, v33) ; v14 = 0, v33 = 128 +;; @003b jump block1 +;; +;; block1: +;; @003b v30 = iconst.i64 132 +;; @003b v31 = iadd.i64 v29, v30 ; v30 = 132 +;; @003b store notrap aligned region2 v31, v3 +;; @003b return +;; } +;; +;; function u0:2(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; fn0 = colocated u805306368:12 sig0 +;; fn1 = colocated u805306368:2 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @003d v3 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003d v4 = load.i64 notrap aligned region2 v3 +;; @003d v5 = iconst.i64 1 +;; @003d v6 = iadd v4, v5 ; v5 = 1 +;; @003d v7 = iconst.i64 0 +;; @003d v8 = icmp sge v6, v7 ; v7 = 0 +;; @003d brif v8, block2, block3(v6) +;; +;; block2: +;; v50 = iadd.i64 v4, v5 ; v5 = 1 +;; @003d store notrap aligned region2 v50, v3 +;; @003d v10 = call fn0(v0) +;; @003d v12 = load.i64 notrap aligned region2 v3 +;; @003d jump block3(v12) +;; +;; block3(v16: i64): +;; @0045 v17 = iconst.i64 4100 +;; @0045 v18 = iadd v16, v17 ; v17 = 4100 +;; v51 = iconst.i64 0 +;; v52 = icmp sge v18, v51 ; v51 = 0 +;; @0045 brif v52, block4, block5(v18) +;; +;; block4: +;; v53 = iadd.i64 v16, v17 ; v17 = 4100 +;; @0045 store notrap aligned region2 v53, v3 +;; @0045 v22 = call fn0(v0) +;; @0045 v24 = load.i64 notrap aligned region2 v3 +;; @0045 jump block5(v24) +;; +;; block5(v39: i64): +;; @0045 v25 = load.i64 notrap aligned region4 v0+64 +;; @0045 v26 = uextend.i64 v2 +;; v40 = iconst.i64 4096 +;; @0045 v30 = iadd v26, v40 ; v40 = 4096 +;; @0045 v31 = icmp ugt v30, v25 +;; @0045 trapnz v31, heap_oob +;; @0045 v32 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0045 v36 = iadd v32, v26 +;; @0040 v14 = iconst.i32 0 +;; @0045 call fn1(v0, v36, v14, v40) ; v14 = 0, v40 = 4096 +;; @0048 jump block1 +;; +;; block1: +;; @0048 store.i64 notrap aligned region2 v39, v3 +;; @0048 return +;; } diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.wat index 45e72b7043d4..29104ad29719 100644 --- a/tests/disas/memory-copy-fuel.wat +++ b/tests/disas/memory-copy-fuel.wat @@ -33,110 +33,44 @@ ;; @001e brif v10, block2, block3(v8) ;; ;; block2: -;; v106 = iadd.i64 v6, v7 ; v7 = 1 -;; @001e store notrap aligned region2 v106, v5 +;; v61 = iadd.i64 v6, v7 ; v7 = 1 +;; @001e store notrap aligned region2 v61, v5 ;; @001e v12 = call fn0(v0) ;; @001e v14 = load.i64 notrap aligned region2 v5 ;; @001e jump block3(v14) ;; -;; block3(v43: i64): -;; @0025 v18 = load.i64 notrap aligned region4 v0+64 -;; @0025 v19 = uextend.i64 v2 -;; @0025 v20 = uextend.i64 v4 -;; @0025 v23 = iadd v19, v20 -;; @0025 v24 = icmp ugt v23, v18 -;; @0025 trapnz v24, heap_oob -;; @0025 v31 = uextend.i64 v3 -;; @0025 v35 = iadd v31, v20 -;; @0025 v36 = icmp ugt v35, v18 -;; @0025 trapnz v36, heap_oob -;; @0025 v25 = load.i64 notrap aligned readonly can_move region3 v0+56 -;; @0025 v41 = iadd v25, v31 -;; @0025 v29 = iadd v25, v19 -;; @0025 v47 = icmp ugt v41, v29 -;; @0025 brif v47, block6, block7 -;; -;; block4(v49: i64, v50: i64, v51: i64, v52: i64): -;; @0025 v53 = iadd v52, v116 ; v116 = 0x0800_0000 -;; v120 = iconst.i64 0 -;; v121 = icmp sge v53, v120 ; v120 = 0 -;; @0025 brif v121, block8, block9(v53) -;; -;; block5(v89: i64, v90: i64, v91: i64, v92: i64): -;; @0025 v94 = iadd v92, v91 -;; v123 = iconst.i64 0 -;; v124 = icmp sge v94, v123 ; v123 = 0 -;; @0025 brif v124, block14, block15(v94) -;; -;; block6: -;; v116 = iconst.i64 0x0800_0000 -;; v117 = icmp.i64 ugt v20, v116 ; v116 = 0x0800_0000 -;; v118 = iconst.i64 4 -;; v119 = iadd.i64 v43, v118 ; v118 = 4 -;; @0025 brif v117, block4(v29, v41, v20, v119), block5(v29, v41, v20, v119) -;; -;; block8: -;; v122 = iadd.i64 v52, v116 ; v116 = 0x0800_0000 -;; @0025 store notrap aligned region2 v122, v5 -;; @0025 v57 = call fn0(v0) -;; @0025 v59 = load.i64 notrap aligned region2 v5 -;; @0025 jump block9(v59) -;; -;; block9(v64: i64): -;; @0025 call fn1(v0, v49, v50, v116) ; v116 = 0x0800_0000 -;; @0025 v62 = isub.i64 v51, v116 ; v116 = 0x0800_0000 -;; @0025 v63 = icmp ugt v62, v116 ; v116 = 0x0800_0000 -;; @0025 v60 = iadd.i64 v49, v116 ; v116 = 0x0800_0000 -;; @0025 v61 = iadd.i64 v50, v116 ; v116 = 0x0800_0000 -;; @0025 brif v63, block4(v60, v61, v62, v64), block5(v60, v61, v62, v64) -;; -;; block7: -;; @0025 v46 = iconst.i64 0x0800_0000 -;; @0025 v67 = icmp.i64 ugt v20, v46 ; v46 = 0x0800_0000 -;; @0025 v65 = iadd.i64 v29, v20 -;; @0025 v66 = iadd.i64 v41, v20 -;; @0025 v44 = iconst.i64 4 -;; @0025 v45 = iadd.i64 v43, v44 ; v44 = 4 -;; @0025 brif v67, block10(v65, v66, v20, v45), block11(v65, v66, v20, v45) -;; -;; block10(v68: i64, v69: i64, v70: i64, v73: i64): -;; v107 = iconst.i64 0x0800_0000 -;; v108 = iadd v73, v107 ; v107 = 0x0800_0000 -;; v109 = iconst.i64 0 -;; v110 = icmp sge v108, v109 ; v109 = 0 -;; @0025 brif v110, block12, block13(v108) -;; -;; block12: -;; @0025 store.i64 notrap aligned region2 v108, v5 -;; @0025 v78 = call fn0(v0) -;; @0025 v80 = load.i64 notrap aligned region2 v5 -;; @0025 jump block13(v80) -;; -;; block13(v83: i64): -;; v111 = iconst.i64 0x0800_0000 -;; v112 = isub.i64 v68, v111 ; v111 = 0x0800_0000 -;; v113 = isub.i64 v69, v111 ; v111 = 0x0800_0000 -;; @0025 call fn1(v0, v112, v113, v111) ; v111 = 0x0800_0000 -;; v114 = isub.i64 v70, v111 ; v111 = 0x0800_0000 -;; v115 = icmp ugt v114, v111 ; v111 = 0x0800_0000 -;; @0025 brif v115, block10(v112, v113, v114, v83), block11(v112, v113, v114, v83) -;; -;; block11(v84: i64, v85: i64, v86: i64, v93: i64): -;; @0025 v87 = isub v84, v86 -;; @0025 v88 = isub v85, v86 -;; @0025 jump block5(v87, v88, v86, v93) -;; -;; block14: -;; @0025 store.i64 notrap aligned region2 v94, v5 -;; @0025 v98 = call fn0(v0) -;; @0025 v100 = load.i64 notrap aligned region2 v5 -;; @0025 jump block15(v100) -;; -;; block15(v102: i64): -;; @0025 call fn1(v0, v89, v90, v91) +;; block3(v21: i64): +;; @0025 v22 = iconst.i64 4 +;; @0025 v23 = iadd v21, v22 ; v22 = 4 +;; @0025 v18 = uextend.i64 v4 +;; @0025 v24 = iadd v23, v18 +;; v62 = iconst.i64 0 +;; v63 = icmp sge v24, v62 ; v62 = 0 +;; @0025 brif v63, block4, block5(v24) +;; +;; block4: +;; @0025 store.i64 notrap aligned region2 v24, v5 +;; @0025 v28 = call fn0(v0) +;; @0025 v30 = load.i64 notrap aligned region2 v5 +;; @0025 jump block5(v30) +;; +;; block5(v57: i64): +;; @0025 v31 = load.i64 notrap aligned region4 v0+64 +;; @0025 v32 = uextend.i64 v2 +;; @0025 v36 = iadd v32, v18 +;; @0025 v37 = icmp ugt v36, v31 +;; @0025 trapnz v37, heap_oob +;; @0025 v44 = uextend.i64 v3 +;; @0025 v48 = iadd v44, v18 +;; @0025 v49 = icmp ugt v48, v31 +;; @0025 trapnz v49, heap_oob +;; @0025 v38 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0025 v42 = iadd v38, v32 +;; @0025 v54 = iadd v38, v44 +;; @0025 call fn1(v0, v42, v54, v18) ;; @0029 jump block1 ;; ;; block1: -;; @0029 store.i64 notrap aligned region2 v102, v5 +;; @0029 store.i64 notrap aligned region2 v57, v5 ;; @0029 return ;; } diff --git a/tests/misc_testsuite/component-model/async/context-in-compositions.wast b/tests/misc_testsuite/component-model/async/context-in-compositions.wast new file mode 100644 index 000000000000..06c0484b9095 --- /dev/null +++ b/tests/misc_testsuite/component-model/async/context-in-compositions.wast @@ -0,0 +1,438 @@ +;;! component_model_async = true +;;! reference_types = true +;;! multi_memory = true + +;; `post-return` observes the callee task's context slots. +(component + (component $A + (core func $get (canon context.get i32 0)) + (core func $set (canon context.set i32 0)) + (core module $M + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (func (export "f'") (param i32) (result i32) + (call $set (i32.const 0xcafe)) + (i32.add (local.get 0) (i32.const 42))) + (func (export "post") (param i32) + (if (i32.ne (call $get) (i32.const 0xcafe)) (then unreachable))) + ) + (core instance $m (instantiate $M (with "" (instance + (export "get" (func $get)) + (export "set" (func $set)))))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'") (post-return (core func $m "post")))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $set (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "set" (func $set (param i32))) + (func (export "g'") (result i32) + (call $set (i32.const 0x1234)) + (call $f' (i32.const 1234))) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "set" (func $set)))))) + (func (export "g") (result u32) (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 1276)) + +;; A `context.set` performed by `post-return` belongs to the callee's task +;; and must not leak back out into the caller. +(component + (component $A + (core func $set (canon context.set i32 0)) + (core module $M + (import "" "set" (func $set (param i32))) + (func (export "f'") (param i32) (result i32) + (i32.add (local.get 0) (i32.const 42))) + (func (export "post") (param i32) + (call $set (i32.const 0xbeef))) + ) + (core instance $m (instantiate $M (with "" (instance + (export "set" (func $set)))))) + (func (export "f") (param "x" u32) (result u32) + (canon lift (core func $m "f'") (post-return (core func $m "post")))) + ) + + (component $B + (import "f" (func $f (param "x" u32) (result u32))) + (core func $f' (canon lower (func $f))) + (core func $get (canon context.get i32 0)) + (core func $set (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32) (result i32))) + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (func (export "g'") (result i32) (local $r i32) + (call $set (i32.const 0x1234)) + (local.set $r (call $f' (i32.const 1234))) + (if (i32.ne (call $get) (i32.const 0x1234)) (then unreachable)) + (local.get $r)) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "get" (func $get)) + (export "set" (func $set)))))) + (func (export "g") (result u32) (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 1276)) + +;; The callee's `realloc`, invoked by the adapter to lower the string parameters +;; into the callee's memory, runs with fresh context slots on every call: what +;; one call stores is neither visible to the next call nor to the callee +;; function itself. +(component + (component $A + (core func $get (canon context.get i32 0)) + (core func $set (canon context.set i32 0)) + (core module $M + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (memory (export "memory") 1) + (global $bump (mut i32) (i32.const 8)) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + (local $ret i32) + (if (i32.ne (call $get) (i32.const 0)) (then unreachable)) + (call $set (i32.const 0x9999)) + (local.set $ret (global.get $bump)) + (global.set $bump (i32.and + (i32.add (i32.add (global.get $bump) (local.get 3)) (i32.const 7)) + (i32.const -8))) + (local.get $ret)) + (func (export "f'") (param i32 i32 i32 i32) + (if (i32.ne (call $get) (i32.const 0)) (then unreachable))) + ) + (core instance $m (instantiate $M (with "" (instance + (export "get" (func $get)) + (export "set" (func $set)))))) + (func (export "f") (param "x" string) (param "y" string) + (canon lift (core func $m "f'") + (memory (core memory $m "memory")) + (realloc (core func $m "realloc")))) + ) + + (component $B + (import "f" (func $f (param "x" string) (param "y" string))) + (core module $Libc + (memory (export "memory") 1) + (data (i32.const 16) "hello")) + (core instance $libc (instantiate $Libc)) + (core func $f' (canon lower (func $f) + (memory (core memory $libc "memory")))) + (core func $set (canon context.set i32 0)) + (core module $N + (import "" "f'" (func $f' (param i32 i32 i32 i32))) + (import "" "set" (func $set (param i32))) + (func (export "g'") (result i32) + (call $set (i32.const 0x1234)) + (call $f' (i32.const 16) (i32.const 5) (i32.const 16) (i32.const 5)) + (i32.const 100)) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "set" (func $set)))))) + (func (export "g") (result u32) (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 100)) + +;; The caller's `realloc`, invoked by the adapter to lower the string result +;; back into the caller's memory, also runs with fresh context slots -- it must +;; not see the caller task's slots. +(component + (component $A + (core module $M + (memory (export "memory") 1) + (data (i32.const 16) "hello") + (func (export "f'") (result i32) + (i32.store (i32.const 8) (i32.const 16)) + (i32.store (i32.const 12) (i32.const 5)) + (i32.const 8)) + ) + (core instance $m (instantiate $M)) + (func (export "f") (result string) + (canon lift (core func $m "f'") (memory (core memory $m "memory")))) + ) + + (component $B + (import "f" (func $f (result string))) + (core func $get (canon context.get i32 0)) + (core func $set (canon context.set i32 0)) + (core module $Libc + (import "" "get" (func $get (result i32))) + (memory (export "memory") 1) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + (if (i32.ne (call $get) (i32.const 0)) (then unreachable)) + (i32.const 64)) + ) + (core instance $libc (instantiate $Libc (with "" (instance + (export "get" (func $get)))))) + (core func $f' (canon lower (func $f) + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")))) + (core module $N + (import "" "f'" (func $f' (param i32))) + (import "" "set" (func $set (param i32))) + (func (export "g'") (result i32) + (call $set (i32.const 0x1234)) + (call $f' (i32.const 8)) + (i32.const 200)) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "set" (func $set)))))) + (func (export "g") (result u32) (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 200)) + +;; A `context.set` performed by the caller's `realloc` is discarded when it +;; returns rather than clobbering the caller task's slots. +(component + (component $A + (core module $M + (memory (export "memory") 1) + (data (i32.const 16) "hello") + (func (export "f'") (result i32) + (i32.store (i32.const 8) (i32.const 16)) + (i32.store (i32.const 12) (i32.const 5)) + (i32.const 8)) + ) + (core instance $m (instantiate $M)) + (func (export "f") (result string) + (canon lift (core func $m "f'") (memory (core memory $m "memory")))) + ) + + (component $B + (import "f" (func $f (result string))) + (core func $get (canon context.get i32 0)) + (core func $set (canon context.set i32 0)) + (core module $Libc + (import "" "set" (func $set (param i32))) + (memory (export "memory") 1) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + (call $set (i32.const 0x7777)) + (i32.const 64)) + ) + (core instance $libc (instantiate $Libc (with "" (instance + (export "set" (func $set)))))) + (core func $f' (canon lower (func $f) + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")))) + (core module $N + (import "" "f'" (func $f' (param i32))) + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (func (export "g'") (result i32) + (call $set (i32.const 0x1234)) + (call $f' (i32.const 8)) + (if (i32.ne (call $get) (i32.const 0x1234)) (then unreachable)) + (i32.const 300)) + ) + (core instance $n (instantiate $N (with "" (instance + (export "f'" (func $f')) + (export "get" (func $get)) + (export "set" (func $set)))))) + (func (export "g") (result u32) (canon lift (core func $n "g'"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "f" (func $a "f")))) + (export "g" (func $b "g")) +) +(assert_return (invoke "g") (u32.const 300)) + +;; Similar to above, but permuting async in signatures. +(component + (component $A + (core func $get (canon context.get i32 0)) + (core func $set (canon context.set i32 0)) + + (core module $Libc + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (memory (export "memory") 1) + (global $bump (mut i32) (i32.const 16)) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + (local $ret i32) + ;; Fresh task for every `realloc`, no matter which adapter drives it. + (if (i32.ne (call $get) (i32.const 0)) (then unreachable)) + (call $set (i32.const 0xa1)) + (local.set $ret (global.get $bump)) + (global.set $bump (i32.and + (i32.add (i32.add (global.get $bump) (local.get 3)) (i32.const 7)) + (i32.const -8))) + (local.get $ret)) + ) + (core instance $libc (instantiate $Libc (with "" (instance + (export "get" (func $get)) + (export "set" (func $set)))))) + + (core func $task.return (canon task.return (result string) + (memory (core memory $libc "memory")))) + + (core module $M + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (import "" "task.return" (func $task.return (param i32 i32))) + (import "" "memory" (memory 1)) + + (func (export "f-sync") (param $ptr i32) (param $len i32) (result i32) + (if (i32.ne (call $get) (i32.const 0)) (then unreachable)) + (call $set (i32.const 0xc0de)) + (i32.store (i32.const 8) (local.get $ptr)) + (i32.store (i32.const 12) (local.get $len)) + (i32.const 8)) + (func (export "f-sync-post") (param i32) + (if (i32.ne (call $get) (i32.const 0xc0de)) (then unreachable)) + (call $set (i32.const 0xbad))) + + (func (export "f-async") (param $ptr i32) (param $len i32) (result i32) + (if (i32.ne (call $get) (i32.const 0)) (then unreachable)) + (call $set (i32.const 0xc0de)) + (call $task.return (local.get $ptr) (local.get $len)) + (i32.const 0 (; CALLBACK_CODE_EXIT ;))) + (func (export "f-async-cb") (param i32 i32 i32) (result i32) unreachable) + ) + (core instance $m (instantiate $M (with "" (instance + (export "get" (func $get)) + (export "set" (func $set)) + (export "task.return" (func $task.return)) + (export "memory" (memory $libc "memory")))))) + + (func (export "sync-lift") async (param "x" string) (result string) + (canon lift (core func $m "f-sync") + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")) + (post-return (core func $m "f-sync-post")))) + (func (export "async-lift") async (param "x" string) (result string) + (canon lift (core func $m "f-async") + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")) + async + (callback (core func $m "f-async-cb")))) + ) + + (component $B + (import "a" (instance $a + (export "sync-lift" (func async (param "x" string) (result string))) + (export "async-lift" (func async (param "x" string) (result string))))) + + (core func $get (canon context.get i32 0)) + (core func $set (canon context.set i32 0)) + + (core module $Libc + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (memory (export "memory") 1) + (data (i32.const 16) "hello") + (global $bump (mut i32) (i32.const 64)) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + (local $ret i32) + (if (i32.ne (call $get) (i32.const 0)) (then unreachable)) + (call $set (i32.const 0xb1)) + (local.set $ret (global.get $bump)) + (global.set $bump (i32.and + (i32.add (i32.add (global.get $bump) (local.get 3)) (i32.const 7)) + (i32.const -8))) + (local.get $ret)) + ) + (core instance $libc (instantiate $Libc (with "" (instance + (export "get" (func $get)) + (export "set" (func $set)))))) + + (core func $sync-to-sync (canon lower (func $a "sync-lift") + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")))) + (core func $sync-to-async (canon lower (func $a "async-lift") + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")))) + (core func $async-to-sync (canon lower (func $a "sync-lift") async + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")))) + (core func $async-to-async (canon lower (func $a "async-lift") async + (memory (core memory $libc "memory")) + (realloc (core func $libc "realloc")))) + + (core module $M + (import "" "get" (func $get (result i32))) + (import "" "set" (func $set (param i32))) + (import "" "sync-to-sync" (func $sync-to-sync (param i32 i32 i32))) + (import "" "sync-to-async" (func $sync-to-async (param i32 i32 i32))) + (import "" "async-to-sync" (func $async-to-sync (param i32 i32 i32) (result i32))) + (import "" "async-to-async" (func $async-to-async (param i32 i32 i32) (result i32))) + + (func (export "sync-to-sync") + (call $set (i32.const 0x1234)) + (call $sync-to-sync (i32.const 16) (i32.const 5) (i32.const 8)) + (if (i32.ne (call $get) (i32.const 0x1234)) (then unreachable))) + + (func (export "sync-to-async") + (call $set (i32.const 0x1235)) + (call $sync-to-async (i32.const 16) (i32.const 5) (i32.const 8)) + (if (i32.ne (call $get) (i32.const 0x1235)) (then unreachable))) + + (func (export "async-to-sync") + (call $set (i32.const 0x1236)) + (if (i32.ne + (call $async-to-sync (i32.const 16) (i32.const 5) (i32.const 8)) + (i32.const 2 (; RETURNED ;))) + (then unreachable)) + (if (i32.ne (call $get) (i32.const 0x1236)) (then unreachable))) + + (func (export "async-to-async") + (call $set (i32.const 0x1237)) + (if (i32.ne + (call $async-to-async (i32.const 16) (i32.const 5) (i32.const 8)) + (i32.const 2 (; RETURNED ;))) + (then unreachable)) + (if (i32.ne (call $get) (i32.const 0x1237)) (then unreachable))) + ) + (core instance $m (instantiate $M (with "" (instance + (export "get" (func $get)) + (export "set" (func $set)) + (export "sync-to-sync" (func $sync-to-sync)) + (export "sync-to-async" (func $sync-to-async)) + (export "async-to-sync" (func $async-to-sync)) + (export "async-to-async" (func $async-to-async)))))) + + (func (export "sync-to-sync") async (canon lift (core func $m "sync-to-sync"))) + (func (export "sync-to-async") async (canon lift (core func $m "sync-to-async"))) + (func (export "async-to-sync") async (canon lift (core func $m "async-to-sync"))) + (func (export "async-to-async") async (canon lift (core func $m "async-to-async"))) + ) + + (instance $a (instantiate $A)) + (instance $b (instantiate $B (with "a" (instance $a)))) + (export "sync-to-sync" (func $b "sync-to-sync")) + (export "sync-to-async" (func $b "sync-to-async")) + (export "async-to-sync" (func $b "async-to-sync")) + (export "async-to-async" (func $b "async-to-async")) +) +(assert_return (invoke "sync-to-sync")) +(assert_return (invoke "sync-to-async")) +(assert_return (invoke "async-to-sync")) +(assert_return (invoke "async-to-async")) diff --git a/tests/misc_testsuite/component-model/async/task-builtins.wast b/tests/misc_testsuite/component-model/async/task-builtins.wast index e2910f9acc3e..1ec14fd8b997 100644 --- a/tests/misc_testsuite/component-model/async/task-builtins.wast +++ b/tests/misc_testsuite/component-model/async/task-builtins.wast @@ -272,7 +272,7 @@ (if (i32.ne (local.get 2) (i32.const 1)) (then (unreachable))) (if (i32.ne (local.get 3) (i32.const 2)) (then (unreachable))) - (if (i32.ne (call $context.get) (i32.const 400)) (then (unreachable))) + (if (i32.ne (call $context.get) (i32.const 0)) (then (unreachable))) (call $context.set (i32.const 500)) call $backpressure.inc @@ -326,19 +326,19 @@ ;; set this tasks's context before calling $run, in calling $run the ;; runtime will then call `realloc` above for the string return value - ;; which should see our 400 value. That will then set 500 which we should - ;; then see after the return. + ;; which should NOT see our 400 value. That will then set 500 which we + ;; should NOT then see after the return. (func (export "sync-to-sync") (call $context.set (i32.const 400)) (call $sync-to-sync (i32.const 20)) - (if (i32.ne (call $context.get) (i32.const 500)) (then (unreachable))) + (if (i32.ne (call $context.get) (i32.const 400)) (then (unreachable))) ) (func (export "sync-to-async") (call $context.set (i32.const 400)) (call $sync-to-async (i32.const 20)) - (if (i32.ne (call $context.get) (i32.const 500)) (then (unreachable))) + (if (i32.ne (call $context.get) (i32.const 400)) (then (unreachable))) ) (func (export "async-to-sync") @@ -350,7 +350,7 @@ ) (then (unreachable)) ) - (if (i32.ne (call $context.get) (i32.const 500)) (then (unreachable))) + (if (i32.ne (call $context.get) (i32.const 400)) (then (unreachable))) ) (func (export "async-to-async") @@ -362,7 +362,7 @@ ) (then (unreachable)) ) - (if (i32.ne (call $context.get) (i32.const 500)) (then (unreachable))) + (if (i32.ne (call $context.get) (i32.const 400)) (then (unreachable))) ) ) (core instance $m (instantiate $M (with "" (instance