WIP markdown parser text preservation property test

`parse_block` was consuming events that it doesn't handle. This was
fine in its use in `parse_document`, but in its use in `parse_list`
this breaks when there is an empty list item, causing it to consume
list end tags / list item starts / etc.

Cargo.lock

@@ -2510,6 +2510,35 @@ dependencies = [
  "objc",
 ]
 
+[[package]]
+name = "code-product"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9feea482d196b435bb8857c2ec1274926993bc3342953515620eb9641337ee01"
+dependencies = [
+ "code-product-lib",
+ "code-product-macro",
+]
+
+[[package]]
+name = "code-product-lib"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5938681371198e7a690aa008843d39bd3b3b65f17b0101518c388f453c0aaf4e"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "code-product-macro"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0371403e32bb895fc60f96b3b76a46a17b99a9ff85b3dbff0e1c5ff0f1ec8e13"
+dependencies = [
+ "code-product-lib",
+ "proc-macro2",
+]
+
 [[package]]
 name = "codespan-reporting"
 version = "0.11.1"
@@ -2787,6 +2816,12 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
 
+[[package]]
+name = "constptr"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f6b816ec3d57d4febea032f3ce2bf9dca119f999a865f9f24d7020d9eb5167e"
+
 [[package]]
 name = "context_servers"
 version = "0.1.0"
@@ -3001,6 +3036,17 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "cowstr"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613de46416b29f30c0c581b5f4cd7dfc01991f59c416b30aa85a0a82af7c791e"
+dependencies = [
+ "code-product",
+ "constptr",
+ "mutants",
+]
+
 [[package]]
 name = "cpal"
 version = "0.15.3"
@@ -5759,7 +5805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904"
 dependencies = [
  "byteorder-lite",
- "quick-error",
+ "quick-error 2.0.1",
 ]
 
 [[package]]
@@ -6758,13 +6804,17 @@ dependencies = [
  "anyhow",
  "async-recursion 1.1.1",
  "collections",
+ "cowstr",
  "editor",
  "gpui",
  "language",
  "linkify",
  "log",
  "pretty_assertions",
+ "proptest",
  "pulldown-cmark 0.12.1",
+ "pulldown-cmark-to-cmark",
+ "rand 0.8.5",
  "settings",
  "theme",
  "ui",
@@ -7062,6 +7112,12 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
+[[package]]
+name = "mutants"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc0287524726960e07b119cebd01678f852f147742ae0d925e6a520dca956126"
+
 [[package]]
 name = "naga"
 version = "22.1.0"
@@ -8547,6 +8603,26 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "proptest"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d"
+dependencies = [
+ "bit-set 0.5.3",
+ "bit-vec 0.6.3",
+ "bitflags 2.6.0",
+ "lazy_static",
+ "num-traits",
+ "rand 0.8.5",
+ "rand_chacha 0.3.1",
+ "rand_xorshift",
+ "regex-syntax 0.8.4",
+ "rusty-fork",
+ "tempfile",
+ "unarray",
+]
+
 [[package]]
 name = "prost"
 version = "0.9.0"
@@ -8675,6 +8751,15 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3"
 
+[[package]]
+name = "pulldown-cmark-to-cmark"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e02b63adcb49f2eb675b1694b413b3e9fedbf549dfe2cc98727ad97a0c30650"
+dependencies = [
+ "pulldown-cmark 0.12.1",
+]
+
 [[package]]
 name = "qoi"
 version = "0.4.1"
@@ -8684,6 +8769,12 @@ dependencies = [
  "bytemuck",
 ]
 
+[[package]]
+name = "quick-error"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+
 [[package]]
 name = "quick-error"
 version = "2.0.1"
@@ -8859,6 +8950,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_xorshift"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f"
+dependencies = [
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "rangemap"
 version = "1.5.1"
@@ -8909,7 +9009,7 @@ dependencies = [
  "avif-serialize",
  "imgref",
  "loop9",
- "quick-error",
+ "quick-error 2.0.1",
  "rav1e",
  "rgb",
 ]
@@ -9802,6 +9902,18 @@ version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
 
+[[package]]
+name = "rusty-fork"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f"
+dependencies = [
+ "fnv",
+ "quick-error 1.2.3",
+ "tempfile",
+ "wait-timeout",
+]
+
 [[package]]
 name = "rustybuzz"
 version = "0.14.1"
@@ -12610,6 +12722,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "unarray"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+
 [[package]]
 name = "unicase"
 version = "2.7.0"
@@ -12979,6 +13097,15 @@ dependencies = [
  "quote",
 ]
 
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "waker-fn"
 version = "1.2.0"

Cargo.toml

@@ -343,6 +343,7 @@ cocoa = "0.26"
 convert_case = "0.6.0"
 core-foundation = "0.9.3"
 core-foundation-sys = "0.8.6"
+cowstr = "1.3.0"
 ctor = "0.2.6"
 dashmap = "6.0"
 derive_more = "0.99.17"
@@ -382,10 +383,12 @@ pathdiff = "0.2"
 postage = { version = "0.5", features = ["futures-traits"] }
 pretty_assertions = "1.3.0"
 profiling = "1"
+proptest = "1.5.0"
 prost = "0.9"
 prost-build = "0.9"
 prost-types = "0.9"
 pulldown-cmark = { version = "0.12.0", default-features = false }
+pulldown-cmark-to-cmark = "18.0.0"
 rand = "0.8.5"
 regex = "1.5"
 repair_json = "0.1.0"

crates/gpui/src/platform/linux/x11/client.rs

@@ -824,7 +824,7 @@ impl X11Client {
             Event::XkbStateNotify(event) => {
                 let mut state = self.0.borrow_mut();
                 state.xkb.update_mask(
-                    dbg!(event).base_mods.into(),
+                    event.base_mods.into(),
                     event.latched_mods.into(),
                     event.locked_mods.into(),
                     event.base_group as u32,
@@ -836,7 +836,7 @@ impl X11Client {
                     latched_layout: event.latched_group as u32,
                     locked_layout: event.locked_group.into(),
                 };
-                let modifiers = dbg!(Modifiers::from_xkb(&state.xkb));
+                let modifiers = Modifiers::from_xkb(&state.xkb);
                 if state.modifiers == modifiers {
                     drop(state);
                 } else {
@@ -919,7 +919,7 @@ impl X11Client {
                 }));
             }
             Event::KeyRelease(event) => {
-                let window = self.get_window(dbg!(event).event)?;
+                let window = self.get_window(event.event)?;
                 let mut state = self.0.borrow_mut();
 
                 let modifiers = modifiers_from_state(event.state);

crates/gpui/src/window.rs

@@ -3216,11 +3216,24 @@ impl<'a> WindowContext<'a> {
             self.draw();
         }
 
+        let node_id = self
+            .window
+            .focus
+            .and_then(|focus_id| {
+                self.window
+                    .rendered_frame
+                    .dispatch_tree
+                    .focusable_node_id(focus_id)
+            })
+            .unwrap_or_else(|| self.window.rendered_frame.dispatch_tree.root_node_id());
+
         let dispatch_path = self
             .window
             .rendered_frame
             .dispatch_tree
-            .dispatch_path(self.focused_node_id());
+            .dispatch_path(node_id);
+
+        let mut keystroke: Option<Keystroke> = None;
 
         if let Some(event) = event.downcast_ref::<ModifiersChangedEvent>() {
             if event.modifiers.number_of_modifiers() == 0
@@ -3236,36 +3249,30 @@ impl<'a> WindowContext<'a> {
                     _ => None,
                 };
                 if let Some(key) = key {
-                    let keystroke = Keystroke {
+                    keystroke = Some(Keystroke {
                         key: key.to_string(),
                         ime_key: None,
                         modifiers: Modifiers::default(),
-                    };
-                    self.finish_dispatch_keystroke(keystroke, &dispatch_path);
+                    });
                 }
             }
 
             if self.window.pending_modifier.modifiers.number_of_modifiers() == 0
                 && event.modifiers.number_of_modifiers() == 1
             {
-                self.window.pending_modifier.saw_keystroke = false;
+                self.window.pending_modifier.saw_keystroke = false
             }
-            self.window.pending_modifier.modifiers = event.modifiers;
-
-            self.finish_dispatch_key_event(event, &dispatch_path);
+            self.window.pending_modifier.modifiers = event.modifiers
         } else if let Some(key_down_event) = event.downcast_ref::<KeyDownEvent>() {
             self.window.pending_modifier.saw_keystroke = true;
-            self.finish_dispatch_keystroke(key_down_event.keystroke.clone(), &dispatch_path);
-        } else {
-            self.finish_dispatch_key_event(event, &dispatch_path);
+            keystroke = Some(key_down_event.keystroke.clone());
         }
-    }
 
-    fn finish_dispatch_keystroke(
-        &self,
-        keystroke: Keystroke,
-        dispatch_path: &SmallVec<[DispatchNodeId; 32]>,
-    ) {
+        let Some(keystroke) = keystroke else {
+            self.finish_dispatch_key_event(event, dispatch_path);
+            return;
+        };
+
         let mut currently_pending = self.window.pending_input.take().unwrap_or_default();
         if currently_pending.focus.is_some() && currently_pending.focus != self.window.focus {
             currently_pending = PendingInput::default();
@@ -3299,8 +3306,7 @@ impl<'a> WindowContext<'a> {
                         .window
                         .rendered_frame
                         .dispatch_tree
-                        // FIXME: valid?
-                        .dispatch_path(cx.focused_node_id());
+                        .dispatch_path(node_id);
 
                     let to_replay = cx
                         .window
@@ -3328,13 +3334,13 @@ impl<'a> WindowContext<'a> {
             }
         }
 
-        self.finish_dispatch_key_event(event, &dispatch_path)
+        self.finish_dispatch_key_event(event, dispatch_path)
     }
 
     fn finish_dispatch_key_event(
         &mut self,
         event: &dyn Any,
-        dispatch_path: &SmallVec<[DispatchNodeId; 32]>,
+        dispatch_path: SmallVec<[DispatchNodeId; 32]>,
     ) {
         self.dispatch_key_down_up_event(event, &dispatch_path);
         if !self.propagate_event {

crates/markdown_preview/Cargo.toml

@@ -15,6 +15,7 @@ path = "src/markdown_preview.rs"
 test-support = []
 
 [dependencies]
+rand.workspace = true
 anyhow.workspace = true
 async-recursion.workspace = true
 collections.workspace = true
@@ -31,4 +32,7 @@ ui.workspace = true
 workspace.workspace = true
 
 [dev-dependencies]
+cowstr.workspace = true
 editor = { workspace = true, features = ["test-support"] }
+proptest.workspace = true
+pulldown-cmark-to-cmark.workspace = true

crates/markdown_preview/src/markdown_parser.rs

@@ -6,6 +6,13 @@ use language::LanguageRegistry;
 use pulldown_cmark::{Alignment, Event, Options, Parser, Tag, TagEnd};
 use std::{ops::Range, path::PathBuf, sync::Arc};
 
+#[cfg(test)]
+use cowstr::CowStr;
+#[cfg(test)]
+use proptest;
+#[cfg(test)]
+use proptest::prelude::*;
+
 pub async fn parse_markdown(
     markdown_input: &str,
     file_location_directory: Option<PathBuf>,
@@ -102,6 +109,8 @@ impl<'a> MarkdownParser<'a> {
         while !self.eof() {
             if let Some(block) = self.parse_block().await {
                 self.parsed.extend(block);
+            } else {
+                self.cursor += 1;
             }
         }
         self
@@ -163,20 +172,14 @@ impl<'a> MarkdownParser<'a> {
                     let code_block = self.parse_code_block(language).await;
                     Some(vec![ParsedMarkdownElement::CodeBlock(code_block)])
                 }
-                _ => {
-                    self.cursor += 1;
-                    None
-                }
+                _ => None,
             },
             Event::Rule => {
                 let source_range = source_range.clone();
                 self.cursor += 1;
                 Some(vec![ParsedMarkdownElement::HorizontalRule(source_range)])
             }
-            _ => {
-                self.cursor += 1;
-                None
-            }
+            _ => None,
         }
     }
 
@@ -1000,6 +1003,8 @@ Some other content
         - Inner
         - Inner
   2. Goodbyte
+        - Next item empty
+        -
 * Last
 ",
         )
@@ -1021,8 +1026,10 @@ Some other content
                 list_item(97..116, 3, Ordered(1), vec![p("Goodbyte", 100..108)]),
                 list_item(117..124, 4, Unordered, vec![p("Inner", 119..124)]),
                 list_item(133..140, 4, Unordered, vec![p("Inner", 135..140)]),
-                list_item(143..154, 2, Ordered(2), vec![p("Goodbyte", 146..154)]),
-                list_item(155..161, 1, Unordered, vec![p("Last", 157..161)]),
+                list_item(143..159, 2, Ordered(2), vec![p("Goodbyte", 146..154)]),
+                list_item(160..180, 3, Unordered, vec![p("Next item empty", 165..180)]),
+                list_item(186..190, 3, Unordered, vec![]),
+                list_item(191..197, 1, Unordered, vec![p("Last", 193..197)]),
             ]
         );
     }
@@ -1223,6 +1230,35 @@ fn main() {
         );
     }
 
+    fn arbitrary_events<'a>() -> impl Strategy<Value = Vec<Event<'static>>> {
+        proptest::collection::vec(arbitrary_event(), 1..100)
+    }
+
+    fn arbitrary_event() -> impl Strategy<Value = Event<'static>> {
+        prop_oneof![
+            arbitrary_tag().prop_map(Event::Start),
+            arbitrary_tag_end().prop_map(Event::End),
+            any::<CowStr>().prop_map(Event::Text),
+            any::<CowStr>().prop_map(Event::Code),
+            any::<CowStr>().prop_map(Event::InlineMath),
+            any::<CowStr>().prop_map(Event::DisplayMath),
+            any::<CowStr>().prop_map(Event::Html),
+            any::<CowStr>().prop_map(Event::InlineHtml),
+            any::<CowStr>().prop_map(Event::FootnoteReference),
+            Just(Event::SoftBreak),
+            Just(Event::HardBreak),
+            Just(Event::Rule),
+            any::<bool>().prop_map(Event::TaskListMarker),
+        ]
+    }
+
+    proptest! {
+        #[test]
+        fn test_text_preservation_property(events in arbitrary_events()) {
+            assert_eq!(events, vec![])
+        }
+    }
+
     fn rust_lang() -> Arc<Language> {
         Arc::new(Language::new(
             LanguageConfig {

crates/tab_switcher/src/tab_switcher.rs

@@ -372,7 +372,7 @@ impl PickerDelegate for TabSwitcherDelegate {
             icon.color(git_status_color.unwrap_or_default())
         });
 
-        let indicator = render_item_indicator(tab_match.item.boxed_clone(), None, cx);
+        let indicator = render_item_indicator(tab_match.item.boxed_clone(), cx);
         let indicator_color = if let Some(ref indicator) = indicator {
             indicator.color
         } else {

crates/ui/src/components/indicator.rs

@@ -7,7 +7,6 @@ enum IndicatorKind {
     Dot,
     Bar,
     Icon(AnyIcon),
-    Character(char),
 }
 
 #[derive(IntoElement)]
@@ -38,18 +37,12 @@ impl Indicator {
         }
     }
 
-    pub fn character(character: char) -> Self {
-        Self {
-            kind: IndicatorKind::Character(character),
-            color: Color::Default,
-        }
-    }
-
     pub fn color(mut self, color: Color) -> Self {
         self.color = color;
         self
     }
 }
+
 impl RenderOnce for Indicator {
     fn render(self, cx: &mut WindowContext) -> impl IntoElement {
         let container = div().flex_none();
@@ -67,9 +60,6 @@ impl RenderOnce for Indicator {
                 .h_1p5()
                 .rounded_t_md()
                 .bg(self.color.color(cx)),
-            IndicatorKind::Character(character) => container
-                .text_color(self.color.color(cx))
-                .child(character.to_string()),
         }
     }
 }

crates/workspace/src/pane.rs

@@ -1835,18 +1835,6 @@ impl Pane {
         focus_handle: &FocusHandle,
         cx: &mut ViewContext<'_, Pane>,
     ) -> impl IntoElement {
-        let pending_keystrokes = cx.pending_input_keystrokes().unwrap_or(&[]);
-        let char_to_type = cx.bindings_for_action(&ActivateItem(ix)).iter().find_map(
-            |keybinding| match keybinding.remaining_keystrokes(pending_keystrokes) {
-                Some([keystroke])
-                    if keystroke.modifiers == cx.modifiers() && keystroke.key.len() == 1 =>
-                {
-                    keystroke.key.chars().next()
-                }
-                _ => None,
-            },
-        );
-
         let project_path = item.project_path(cx);
 
         let is_active = ix == self.active_item_index;
@@ -1878,7 +1866,7 @@ impl Pane {
 
         let icon = item.tab_icon(cx);
         let close_side = &ItemSettings::get_global(cx).close_position;
-        let indicator = render_item_indicator(item.boxed_clone(), char_to_type, cx);
+        let indicator = render_item_indicator(item.boxed_clone(), cx);
         let item_id = item.item_id();
         let is_first_item = ix == 0;
         let is_last_item = ix == self.items.len() - 1;
@@ -3009,22 +2997,16 @@ pub fn tab_details(items: &[Box<dyn ItemHandle>], cx: &AppContext) -> Vec<usize>
     tab_details
 }
 
-pub fn render_item_indicator(
-    item: Box<dyn ItemHandle>,
-    char_to_type: Option<char>,
-    cx: &WindowContext,
-) -> Option<Indicator> {
-    if let Some(char) = char_to_type {
-        return Some(Indicator::character(char));
-    }
+pub fn render_item_indicator(item: Box<dyn ItemHandle>, cx: &WindowContext) -> Option<Indicator> {
+    maybe!({
+        let indicator_color = match (item.has_conflict(cx), item.is_dirty(cx)) {
+            (true, _) => Color::Warning,
+            (_, true) => Color::Accent,
+            (false, false) => return None,
+        };
 
-    let indicator_color = match (item.has_conflict(cx), item.is_dirty(cx)) {
-        (true, _) => Color::Warning,
-        (_, true) => Color::Accent,
-        (false, false) => return None,
-    };
-
-    Some(Indicator::dot().color(indicator_color))
+        Some(Indicator::dot().color(indicator_color))
+    })
 }
 
 impl Render for DraggedTab {