Enhance extension HTTP client

- Add support for different HTTP methods in extension requests
- Take an optional body in method.
- Keep request/response bodies as bytes
- Implement conversion functions between internal and extension HTTP types
- Update Gleam extension to use new HTTP client

crates/extension/src/wasm_host/wit/since_v0_0_7.rs

@@ -130,37 +130,74 @@ impl common::Host for WasmState {}
 impl http_client::Host for WasmState {
     async fn fetch(
         &mut self,
-        req: http_client::HttpRequest,
+        extension_request: http_client::HttpRequest,
     ) -> wasmtime::Result<Result<http_client::HttpResponse, String>> {
         maybe!(async {
-            let url = &req.url;
-
-            let mut response = self
-                .host
-                .http_client
-                .get(url, AsyncBody::default(), true)
-                .await?;
-
+            let url = &extension_request.url;
+            let request = convert_request(&extension_request)?;
+            let mut response = self.host.http_client.send(request).await?;
             if response.status().is_client_error() || response.status().is_server_error() {
                 bail!("failed to fetch '{url}': status code {}", response.status())
             }
-
-            let mut body = Vec::new();
-            response
-                .body_mut()
-                .read_to_end(&mut body)
-                .await
-                .with_context(|| format!("failed to read response body from '{url}'"))?;
-
-            Ok(http_client::HttpResponse {
-                body: String::from_utf8(body)?,
-            })
+            let extension_response = convert_response(&mut response).await?;
+            Ok(extension_response)
         })
         .await
         .to_wasmtime_result()
     }
 }
 
+fn convert_method(method: http_client::HttpMethod) -> ::http_client::Method {
+    match method {
+        http_client::HttpMethod::Get => ::http_client::Method::GET,
+        http_client::HttpMethod::Post => ::http_client::Method::POST,
+        http_client::HttpMethod::Put => ::http_client::Method::PUT,
+        http_client::HttpMethod::Delete => ::http_client::Method::DELETE,
+        http_client::HttpMethod::Head => ::http_client::Method::HEAD,
+        http_client::HttpMethod::Options => ::http_client::Method::OPTIONS,
+        http_client::HttpMethod::Patch => ::http_client::Method::PATCH,
+    }
+}
+
+fn convert_request(
+    extension_request: &http_client::HttpRequest,
+) -> Result<::http_client::Request<AsyncBody>, anyhow::Error> {
+    let mut request = ::http_client::Request::builder()
+        .method(convert_method(extension_request.method))
+        .uri(&extension_request.url);
+    for (key, value) in &extension_request.headers {
+        request = request.header(key, value);
+    }
+    let body = extension_request
+        .body
+        .clone()
+        .map(AsyncBody::from)
+        .unwrap_or_default();
+    request.body(body).map_err(anyhow::Error::from)
+}
+
+async fn convert_response(
+    response: &mut ::http_client::Response<AsyncBody>,
+) -> Result<http_client::HttpResponse, anyhow::Error> {
+    let mut extension_response = http_client::HttpResponse {
+        body: Vec::new(),
+        headers: Vec::new(),
+    };
+
+    for (key, value) in response.headers() {
+        extension_response
+            .headers
+            .push((key.to_string(), value.to_str().unwrap_or("").to_string()));
+    }
+
+    response
+        .body_mut()
+        .read_to_end(&mut extension_response.body)
+        .await?;
+
+    Ok(extension_response)
+}
+
 #[async_trait]
 impl nodejs::Host for WasmState {
     async fn node_binary_path(&mut self) -> wasmtime::Result<Result<String, String>> {

crates/extension_api/src/extension_api.rs

@@ -19,7 +19,7 @@ pub use wit::{
         github_release_by_tag_name, latest_github_release, GithubRelease, GithubReleaseAsset,
         GithubReleaseOptions,
     },
-    zed::extension::http_client::{fetch, HttpRequest, HttpResponse},
+    zed::extension::http_client::{fetch, HttpMethod, HttpRequest, HttpResponse},
     zed::extension::nodejs::{
         node_binary_path, npm_install_package, npm_package_installed_version,
         npm_package_latest_version,

crates/extension_api/wit/since_v0.0.7/http-client.wit

@@ -1,14 +1,33 @@
 interface http-client {
     /// An HTTP request.
     record http-request {
+        /// The HTTP method for the request.
+        method: http-method,
         /// The URL to which the request should be made.
         url: string,
+        /// Headers for the request.
+        headers: list<tuple<string, string>>,
+        /// The request body.
+        body: option<list<u8>>,
+    }
+
+    /// HTTP methods.
+    enum http-method {
+        get,
+        post,
+        put,
+        delete,
+        head,
+        options,
+        patch,
     }
 
     /// An HTTP response.
     record http-response {
+        /// The response headers.
+        headers: list<tuple<string, string>>,
         /// The response body.
-        body: string,
+        body: list<u8>,
     }
 
     /// Performs an HTTP request and returns the response.

extensions/gleam/src/gleam.rs

@@ -1,10 +1,10 @@
 mod hexdocs;
 
-use std::fs;
+use std::{fs, io};
 use zed::lsp::CompletionKind;
 use zed::{
-    CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand,
-    SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
+    CodeLabel, CodeLabelSpan, HttpMethod, HttpRequest, KeyValueStore, LanguageServerId,
+    SlashCommand, SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
 };
 use zed_extension_api::{self as zed, Result};
 
@@ -194,6 +194,7 @@ impl zed::Extension for GleamExtension {
                 let module_path = components.map(ToString::to_string).collect::<Vec<_>>();
 
                 let response = zed::fetch(&HttpRequest {
+                    method: HttpMethod::Get,
                     url: format!(
                         "https://hexdocs.pm/{package_name}{maybe_path}",
                         maybe_path = if !module_path.is_empty() {
@@ -202,9 +203,15 @@ impl zed::Extension for GleamExtension {
                             String::new()
                         }
                     ),
+                    headers: vec![(
+                        "User-Agent".to_string(),
+                        "Zed (Gleam Extension)".to_string(),
+                    )],
+                    body: None,
                 })?;
 
-                let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+                let (markdown, _modules) =
+                    convert_hexdocs_to_markdown(&mut io::Cursor::new(response.body))?;
 
                 let mut text = String::new();
                 text.push_str(&markdown);

extensions/gleam/src/hexdocs.rs

@@ -1,6 +1,6 @@
 use std::cell::RefCell;
 use std::collections::BTreeSet;
-use std::io::Read;
+use std::io::{self, Read};
 use std::rc::Rc;
 
 use html_to_markdown::markdown::{
@@ -10,23 +10,36 @@ use html_to_markdown::{
     convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter,
     StartTagOutcome, TagHandler,
 };
-use zed_extension_api::{self as zed, HttpRequest, KeyValueStore, Result};
+use zed_extension_api::{self as zed, HttpMethod, HttpRequest, KeyValueStore, Result};
 
 pub fn index(package: String, database: &KeyValueStore) -> Result<()> {
+    let headers = vec![(
+        "User-Agent".to_string(),
+        "Zed (Gleam Extension)".to_string(),
+    )];
+
     let response = zed::fetch(&HttpRequest {
+        method: HttpMethod::Get,
         url: format!("https://hexdocs.pm/{package}"),
+        headers: headers.clone(),
+        body: None,
     })?;
 
-    let (package_root_markdown, modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+    let (package_root_markdown, modules) =
+        convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
 
     database.insert(&package, &package_root_markdown)?;
 
     for module in modules {
         let response = zed::fetch(&HttpRequest {
+            method: HttpMethod::Get,
             url: format!("https://hexdocs.pm/{package}/{module}.html"),
+            headers: headers.clone(),
+            body: None,
         })?;
 
-        let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+        let (markdown, _modules) =
+            convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
 
         database.insert(&format!("{module} ({package})"), &markdown)?;
     }