Compare commits

...

5 Commits

Author SHA1 Message Date
Piotr Osiewicz
5a120af767 Re-land Cargo.lock 2025-01-09 20:31:21 +01:00
Piotr Osiewicz
6dbed8deeb Merge branch 'main' into wasi-p2 2025-01-09 20:25:53 +01:00
Piotr Osiewicz
0f55ff6f5b Use cargo.lock from main (temporarily) 2025-01-09 20:24:14 +01:00
Piotr Osiewicz
76d8623b86 Update crates/extension/src/extension_builder.rs 2024-11-18 12:59:26 +01:00
Piotr Osiewicz
c0b751be1f extension: Bump WASI to p2 2024-11-18 12:57:56 +01:00
9 changed files with 528 additions and 225 deletions

663
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -512,14 +512,14 @@ url = "2.2"
uuid = { version = "1.1.2", features = ["v4", "v5", "v7", "serde"] }
wasmparser = "0.215"
wasm-encoder = "0.215"
wasmtime = { version = "24", default-features = false, features = [
wasmtime = { version = "26", default-features = false, features = [
"async",
"demangle",
"runtime",
"cranelift",
"component-model",
] }
wasmtime-wasi = "24"
wasmtime-wasi = "26"
which = "6.0.0"
wit-component = "0.201"
zstd = "0.11"

View File

@@ -18,28 +18,19 @@ use wasm_encoder::{ComponentSectionId, Encode as _, RawSection, Section as _};
use wasmparser::Parser;
use wit_component::ComponentEncoder;
/// Currently, we compile with Rust's `wasm32-wasip1` target, which works with WASI `preview1`.
/// But the WASM component model is based on WASI `preview2`. So we need an 'adapter' WASM
/// module, which implements the `preview1` interface in terms of `preview2`.
///
/// Once Rust 1.78 is released, there will be a `wasm32-wasip2` target available, so we will
/// not need the adapter anymore.
const RUST_TARGET: &str = "wasm32-wasip1";
const WASI_ADAPTER_URL: &str =
"https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.reactor.wasm";
const RUST_TARGET: &str = "wasm32-wasip2";
/// Compiling Tree-sitter parsers from C to WASM requires Clang 17, and a WASM build of libc
/// and clang's runtime library. The `wasi-sdk` provides these binaries.
///
/// Once Clang 17 and its wasm target are available via system package managers, we won't need
/// to download this.
const WASI_SDK_URL: &str = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/";
const WASI_SDK_URL: &str = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/";
const WASI_SDK_ASSET_NAME: Option<&str> = if cfg!(target_os = "macos") {
Some("wasi-sdk-21.0-macos.tar.gz")
Some("wasi-sdk-24.0-macos.tar.gz")
} else if cfg!(any(target_os = "linux", target_os = "freebsd")) {
Some("wasi-sdk-21.0-linux.tar.gz")
Some("wasi-sdk-24.0-linux.tar.gz")
} else if cfg!(target_os = "windows") {
Some("wasi-sdk-21.0.m-mingw.tar.gz")
Some("wasi-sdk-24.0.m-mingw.tar.gz")
} else {
None
};
@@ -121,8 +112,6 @@ impl ExtensionBuilder {
options: CompileExtensionOptions,
) -> Result<(), anyhow::Error> {
self.install_rust_wasm_target_if_needed()?;
let adapter_bytes = self.install_wasi_preview1_adapter_if_needed().await?;
let cargo_toml_content = fs::read_to_string(extension_dir.join("Cargo.toml"))?;
let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?;
@@ -130,6 +119,7 @@ impl ExtensionBuilder {
"compiling Rust crate for extension {}",
extension_dir.display()
);
let output = util::command::new_std_command("cargo")
.args(["build", "--target", RUST_TARGET])
.args(options.release.then_some("--release"))
@@ -168,20 +158,12 @@ impl ExtensionBuilder {
let wasm_bytes = fs::read(&wasm_path)
.with_context(|| format!("failed to read output module `{}`", wasm_path.display()))?;
let encoder = ComponentEncoder::default()
.module(&wasm_bytes)?
.adapter("wasi_snapshot_preview1", &adapter_bytes)
.context("failed to load adapter module")?
.validate(true);
log::info!(
"encoding wasm component for extension {}",
extension_dir.display()
);
let component_bytes = encoder
.encode()
.context("failed to encode wasm component")?;
let component_bytes = wasm_bytes;
let component_bytes = self
.strip_custom_sections(&component_bytes)
@@ -379,38 +361,6 @@ impl ExtensionBuilder {
Ok(())
}
async fn install_wasi_preview1_adapter_if_needed(&self) -> Result<Vec<u8>> {
let cache_path = self.cache_dir.join("wasi_snapshot_preview1.reactor.wasm");
if let Ok(content) = fs::read(&cache_path) {
if Parser::is_core_wasm(&content) {
return Ok(content);
}
}
fs::remove_file(&cache_path).ok();
log::info!(
"downloading wasi adapter module to {}",
cache_path.display()
);
let mut response = self
.http
.get(WASI_ADAPTER_URL, AsyncBody::default(), true)
.await?;
let mut content = Vec::new();
let mut body = BufReader::new(response.body_mut());
body.read_to_end(&mut content).await?;
fs::write(&cache_path, &content)
.with_context(|| format!("failed to save file {}", cache_path.display()))?;
if !Parser::is_core_wasm(&content) {
bail!("downloaded wasi adapter is invalid");
}
Ok(content)
}
async fn install_wasi_sdk_if_needed(&self) -> Result<PathBuf> {
let url = if let Some(asset_name) = WASI_SDK_ASSET_NAME {
format!("{WASI_SDK_URL}/{asset_name}")

View File

@@ -84,7 +84,7 @@ impl HostWorktree for WasmState {
latest::HostWorktree::which(self, delegate, binary_name).await
}
fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
Ok(())
}
}

View File

@@ -92,7 +92,7 @@ impl HostWorktree for WasmState {
latest::HostWorktree::which(self, delegate, binary_name).await
}
fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
// We only ever hand out borrows of worktrees.
Ok(())
}

View File

@@ -147,7 +147,7 @@ impl HostWorktree for WasmState {
latest::HostWorktree::which(self, delegate, binary_name).await
}
fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
// We only ever hand out borrows of worktrees.
Ok(())
}

View File

@@ -240,7 +240,7 @@ impl HostKeyValueStore for WasmState {
kv_store.insert(key, value).await.to_wasmtime_result()
}
fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
async fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
// We only ever hand out borrows of key-value stores.
Ok(())
}
@@ -282,7 +282,7 @@ impl HostWorktree for WasmState {
latest::HostWorktree::which(self, delegate, binary_name).await
}
fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
// We only ever hand out borrows of worktrees.
Ok(())
}
@@ -350,7 +350,7 @@ impl http_client::HostHttpResponseStream for WasmState {
.to_wasmtime_result()
}
fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
async fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
Ok(())
}
}

View File

@@ -259,7 +259,7 @@ impl HostKeyValueStore for WasmState {
kv_store.insert(key, value).await.to_wasmtime_result()
}
fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
async fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
// We only ever hand out borrows of key-value stores.
Ok(())
}
@@ -275,7 +275,7 @@ impl HostProject for WasmState {
Ok(project.worktree_ids())
}
fn drop(&mut self, _project: Resource<Project>) -> Result<()> {
async fn drop(&mut self, _project: Resource<Project>) -> Result<()> {
// We only ever hand out borrows of projects.
Ok(())
}
@@ -325,7 +325,7 @@ impl HostWorktree for WasmState {
Ok(delegate.which(binary_name).await)
}
fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
// We only ever hand out borrows of worktrees.
Ok(())
}
@@ -393,7 +393,7 @@ impl http_client::HostHttpResponseStream for WasmState {
.to_wasmtime_result()
}
fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
async fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
Ok(())
}
}

View File

@@ -2,4 +2,4 @@
channel = "1.81"
profile = "minimal"
components = [ "rustfmt", "clippy" ]
targets = [ "x86_64-apple-darwin", "aarch64-apple-darwin", "x86_64-unknown-linux-gnu", "wasm32-wasip1", "x86_64-pc-windows-msvc" ]
targets = [ "x86_64-apple-darwin", "aarch64-apple-darwin", "x86_64-unknown-linux-gnu", "wasm32-wasip2", "x86_64-pc-windows-msvc" ]