Write a rustdoc2md example binary to explore markdown generation

crates/assistant/src/slash_command/rustdoc_command.rs

@@ -37,6 +37,8 @@ impl RustdocSlashCommand {
             if let Ok(contents) = fs.load(&local_cargo_doc_path).await {
                 let (markdown, _items) = convert_rustdoc_to_markdown(contents.as_bytes())?;
 
+                dbg!(_items);
+
                 return Ok((RustdocSource::Local, markdown));
             }
         }

crates/html_to_markdown/src/markdown_writer.rs

@@ -43,6 +43,10 @@ impl MarkdownWriter {
         &self.current_element_stack
     }
 
+    pub fn len(&self) -> usize {
+        self.markdown.len()
+    }
+
     pub fn is_inside(&self, tag: &str) -> bool {
         self.current_element_stack
             .iter()

crates/rustdoc/examples/rustdoc2md.rs

@@ -0,0 +1,47 @@
+use anyhow::Result;
+use rustdoc::{convert_rustdoc_to_markdown, RustdocItem};
+use std::path::{Path, PathBuf};
+
+fn fetch_and_convert_docs(crate_docs_path: &Path) -> Result<(String, Vec<RustdocItem>)> {
+    if !crate_docs_path.exists() {
+        anyhow::bail!("File not found at {:?}", crate_docs_path);
+    }
+
+    let html_content = std::fs::read_to_string(crate_docs_path)?;
+    let (markdown, items) = convert_rustdoc_to_markdown(html_content.as_bytes())?;
+
+    Ok((markdown, items))
+}
+
+fn main() -> Result<()> {
+    let args: Vec<String> = std::env::args().collect();
+    if args.len() != 2 {
+        println!("Usage: {} <path_to_crate_docs>", args[0]);
+        std::process::exit(1);
+    }
+
+    let crate_docs_path = PathBuf::from(&args[1]);
+
+    let (markdown, items) = fetch_and_convert_docs(&crate_docs_path.join("index.html"))?;
+
+    println!("Converted Markdown:\n{}", markdown);
+
+    println!("\nLinked item contents:");
+    for item in &items {
+        println!("- {}", item.href);
+        let item_path = crate_docs_path.join(item.href.as_ref());
+        match fetch_and_convert_docs(&item_path) {
+            Ok((item_markdown, _)) => {
+                println!("{}\n", item_markdown);
+            }
+            Err(e) => {
+                eprintln!(
+                    "Failed to fetch and convert item: {} for path {:?}",
+                    e, item.href
+                );
+            }
+        }
+    }
+
+    Ok(())
+}

crates/rustdoc/src/item.rs

@@ -37,13 +37,17 @@ impl RustdocItemKind {
     }
 }
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)]
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub struct RustdocItem {
     pub kind: RustdocItemKind,
     /// The item path, up until the name of the item.
     pub path: Vec<Arc<str>>,
     /// The name of the item.
     pub name: Arc<str>,
+    /// The target of the link, which contains more information about the item.
+    pub href: Arc<str>,
+    /// The position of the item's text in the markdown output.
+    pub output_range: std::ops::Range<usize>,
 }
 
 impl RustdocItem {

crates/rustdoc/src/to_markdown.rs

@@ -263,7 +263,7 @@ impl RustdocItemCollector {
         }
     }
 
-    fn parse_item(tag: &HtmlElement) -> Option<RustdocItem> {
+    fn parse_item(tag: &HtmlElement, output_offset: usize) -> Option<RustdocItem> {
         if tag.tag() != "a" {
             return None;
         }
@@ -291,6 +291,8 @@ impl RustdocItemCollector {
                         kind,
                         name: name.into(),
                         path: parts.map(Into::into).collect(),
+                        href: href.into(),
+                        output_range: output_offset..output_offset,
                     });
                 }
             }
@@ -321,7 +323,7 @@ impl HandleTag for RustdocItemCollector {
                 });
 
                 if !is_reexport {
-                    if let Some(item) = Self::parse_item(tag) {
+                    if let Some(item) = Self::parse_item(tag, writer.len()) {
                         self.items.insert(item);
                     }
                 }
@@ -331,6 +333,26 @@ impl HandleTag for RustdocItemCollector {
 
         StartTagOutcome::Continue
     }
+
+    fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) {
+        match tag.tag() {
+            "a" => {
+                if let Some(href) = tag.attr("href") {
+                    if self
+                        .items
+                        .last()
+                        .map_or(false, |item| item.href.as_ref() == href)
+                    {
+                        if let Some(mut last_item) = self.items.pop() {
+                            last_item.output_range.end = writer.len();
+                            self.items.insert(last_item);
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
 }
 
 #[cfg(test)]