wip

This PR fixes an issue where the eval was incorrectly pulling the
provider/model from the user settings, which could cause problems when
running certain evals.

Was introduced in #30168 due to the restructuring after the removal of
the `assistant` crate.

Release Notes:

- N/A

crates/agent/src/agent.rs

@@ -117,6 +117,7 @@ pub fn init(
     client: Arc<Client>,
     prompt_builder: Arc<PromptBuilder>,
     language_registry: Arc<LanguageRegistry>,
+    is_eval: bool,
     cx: &mut App,
 ) {
     AssistantSettings::register(cx);
@@ -124,7 +125,11 @@ pub fn init(
 
     assistant_context_editor::init(client.clone(), cx);
     rules_library::init(cx);
-    init_language_model_settings(cx);
+    if !is_eval {
+        // Initializing the language model from the user settings messes with the eval, so we only initialize them when
+        // we're not running inside of the eval.
+        init_language_model_settings(cx);
+    }
     assistant_slash_command::init(cx);
     thread_store::init(cx);
     agent_panel::init(cx);

crates/agent/src/agent_diff.rs

@@ -1348,6 +1348,7 @@ impl AgentDiff {
             ThreadEvent::NewRequest
             | ThreadEvent::Stopped(Ok(StopReason::EndTurn))
             | ThreadEvent::Stopped(Ok(StopReason::MaxTokens))
+            | ThreadEvent::Stopped(Ok(StopReason::Refusal))
             | ThreadEvent::Stopped(Err(_))
             | ThreadEvent::ShowError(_)
             | ThreadEvent::CompletionCanceled => {

crates/agent/src/thread.rs

@@ -1693,6 +1693,43 @@ impl Thread {
                                     project.set_agent_location(None, cx);
                                 });
                             }
+                            StopReason::Refusal => {
+                                thread.project.update(cx, |project, cx| {
+                                    project.set_agent_location(None, cx);
+                                });
+
+                                // Remove the turn that was refused.
+                                //
+                                // https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/handle-streaming-refusals#reset-context-after-refusal
+                                {
+                                    let mut messages_to_remove = Vec::new();
+
+                                    for (ix, message) in thread.messages.iter().enumerate().rev() {
+                                        messages_to_remove.push(message.id);
+
+                                        if message.role == Role::User {
+                                            if ix == 0 {
+                                                break;
+                                            }
+
+                                            if let Some(prev_message) = thread.messages.get(ix - 1) {
+                                                if prev_message.role == Role::Assistant {
+                                                    break;
+                                                }
+                                            }
+                                        }
+                                    }
+
+                                    for message_id in messages_to_remove {
+                                        thread.delete_message(message_id, cx);
+                                    }
+                                }
+
+                                cx.emit(ThreadEvent::ShowError(ThreadError::Message {
+                                    header: "Language model refusal".into(),
+                                    message: "Model refused to generate content for safety reasons.".into(),
+                                }));
+                            }
                         },
                         Err(error) => {
                             thread.project.update(cx, |project, cx| {

crates/anthropic/src/anthropic.rs

@@ -34,7 +34,6 @@ pub enum AnthropicModelMode {
 pub enum Model {
     #[serde(rename = "claude-3-5-sonnet", alias = "claude-3-5-sonnet-latest")]
     Claude3_5Sonnet,
-    #[default]
     #[serde(rename = "claude-3-7-sonnet", alias = "claude-3-7-sonnet-latest")]
     Claude3_7Sonnet,
     #[serde(
@@ -49,6 +48,7 @@ pub enum Model {
         alias = "claude-opus-4-thinking-latest"
     )]
     ClaudeOpus4Thinking,
+    #[default]
     #[serde(rename = "claude-sonnet-4", alias = "claude-sonnet-4-latest")]
     ClaudeSonnet4,
     #[serde(
@@ -150,10 +150,10 @@ impl Model {
 
     pub fn display_name(&self) -> &str {
         match self {
-            Model::ClaudeOpus4 => "Claude 4 Opus",
-            Model::ClaudeOpus4Thinking => "Claude 4 Opus Thinking",
-            Model::ClaudeSonnet4 => "Claude 4 Sonnet",
-            Model::ClaudeSonnet4Thinking => "Claude 4 Sonnet Thinking",
+            Model::ClaudeOpus4 => "Claude Opus 4",
+            Model::ClaudeOpus4Thinking => "Claude Opus 4 Thinking",
+            Model::ClaudeSonnet4 => "Claude Sonnet 4",
+            Model::ClaudeSonnet4Thinking => "Claude Sonnet 4 Thinking",
             Self::Claude3_7Sonnet => "Claude 3.7 Sonnet",
             Self::Claude3_5Sonnet => "Claude 3.5 Sonnet",
             Self::Claude3_7SonnetThinking => "Claude 3.7 Sonnet Thinking",

crates/assistant_context_editor/src/context.rs

@@ -2204,6 +2204,7 @@ impl AssistantContext {
                             StopReason::ToolUse => {}
                             StopReason::EndTurn => {}
                             StopReason::MaxTokens => {}
+                            StopReason::Refusal => {}
                         }
                     }
                 })

crates/bedrock/src/models.rs

@@ -15,6 +15,20 @@ pub enum BedrockModelMode {
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, EnumIter)]
 pub enum Model {
     // Anthropic models (already included)
+    #[serde(rename = "claude-sonnet-4", alias = "claude-sonnet-4-latest")]
+    ClaudeSonnet4,
+    #[serde(
+        rename = "claude-sonnet-4-thinking",
+        alias = "claude-sonnet-4-thinking-latest"
+    )]
+    ClaudeSonnet4Thinking,
+    #[serde(rename = "claude-opus-4", alias = "claude-opus-4-latest")]
+    ClaudeOpus4,
+    #[serde(
+        rename = "claude-opus-4-thinking",
+        alias = "claude-opus-4-thinking-latest"
+    )]
+    ClaudeOpus4Thinking,
     #[default]
     #[serde(rename = "claude-3-5-sonnet-v2", alias = "claude-3-5-sonnet-latest")]
     Claude3_5SonnetV2,
@@ -112,6 +126,12 @@ impl Model {
 
     pub fn id(&self) -> &str {
         match self {
+            Model::ClaudeSonnet4 | Model::ClaudeSonnet4Thinking => {
+                "anthropic.claude-sonnet-4-20250514-v1:0"
+            }
+            Model::ClaudeOpus4 | Model::ClaudeOpus4Thinking => {
+                "anthropic.claude-opus-4-20250514-v1:0"
+            }
             Model::Claude3_5SonnetV2 => "anthropic.claude-3-5-sonnet-20241022-v2:0",
             Model::Claude3_5Sonnet => "anthropic.claude-3-5-sonnet-20240620-v1:0",
             Model::Claude3Opus => "anthropic.claude-3-opus-20240229-v1:0",
@@ -163,6 +183,10 @@ impl Model {
 
     pub fn display_name(&self) -> &str {
         match self {
+            Self::ClaudeSonnet4 => "Claude Sonnet 4",
+            Self::ClaudeSonnet4Thinking => "Claude Sonnet 4 Thinking",
+            Self::ClaudeOpus4 => "Claude Opus 4",
+            Self::ClaudeOpus4Thinking => "Claude Opus 4 Thinking",
             Self::Claude3_5SonnetV2 => "Claude 3.5 Sonnet v2",
             Self::Claude3_5Sonnet => "Claude 3.5 Sonnet",
             Self::Claude3Opus => "Claude 3 Opus",
@@ -219,7 +243,9 @@ impl Model {
             | Self::Claude3Opus
             | Self::Claude3Sonnet
             | Self::Claude3_5Haiku
-            | Self::Claude3_7Sonnet => 200_000,
+            | Self::Claude3_7Sonnet
+            | Self::ClaudeSonnet4
+            | Self::ClaudeOpus4 => 200_000,
             Self::AmazonNovaPremier => 1_000_000,
             Self::PalmyraWriterX5 => 1_000_000,
             Self::PalmyraWriterX4 => 128_000,
@@ -231,7 +257,12 @@ impl Model {
     pub fn max_output_tokens(&self) -> u32 {
         match self {
             Self::Claude3Opus | Self::Claude3Sonnet | Self::Claude3_5Haiku => 4_096,
-            Self::Claude3_7Sonnet | Self::Claude3_7SonnetThinking => 128_000,
+            Self::Claude3_7Sonnet
+            | Self::Claude3_7SonnetThinking
+            | Self::ClaudeSonnet4
+            | Self::ClaudeSonnet4Thinking
+            | Self::ClaudeOpus4
+            | Model::ClaudeOpus4Thinking => 128_000,
             Self::Claude3_5SonnetV2 | Self::PalmyraWriterX4 | Self::PalmyraWriterX5 => 8_192,
             Self::Custom {
                 max_output_tokens, ..
@@ -246,7 +277,11 @@ impl Model {
             | Self::Claude3Opus
             | Self::Claude3Sonnet
             | Self::Claude3_5Haiku
-            | Self::Claude3_7Sonnet => 1.0,
+            | Self::Claude3_7Sonnet
+            | Self::ClaudeOpus4
+            | Self::ClaudeOpus4Thinking
+            | Self::ClaudeSonnet4
+            | Self::ClaudeSonnet4Thinking => 1.0,
             Self::Custom {
                 default_temperature,
                 ..
@@ -264,6 +299,10 @@ impl Model {
             | Self::Claude3_5SonnetV2
             | Self::Claude3_7Sonnet
             | Self::Claude3_7SonnetThinking
+            | Self::ClaudeOpus4
+            | Self::ClaudeOpus4Thinking
+            | Self::ClaudeSonnet4
+            | Self::ClaudeSonnet4Thinking
             | Self::Claude3_5Haiku => true,
 
             // Amazon Nova models (all support tool use)
@@ -289,6 +328,12 @@ impl Model {
             Model::Claude3_7SonnetThinking => BedrockModelMode::Thinking {
                 budget_tokens: Some(4096),
             },
+            Model::ClaudeSonnet4Thinking => BedrockModelMode::Thinking {
+                budget_tokens: Some(4096),
+            },
+            Model::ClaudeOpus4Thinking => BedrockModelMode::Thinking {
+                budget_tokens: Some(4096),
+            },
             _ => BedrockModelMode::Default,
         }
     }
@@ -324,6 +369,10 @@ impl Model {
             (Model::Claude3Opus, "us")
             | (Model::Claude3_5Haiku, "us")
             | (Model::Claude3_7Sonnet, "us")
+            | (Model::ClaudeSonnet4, "us")
+            | (Model::ClaudeOpus4, "us")
+            | (Model::ClaudeSonnet4Thinking, "us")
+            | (Model::ClaudeOpus4Thinking, "us")
             | (Model::Claude3_7SonnetThinking, "us")
             | (Model::AmazonNovaPremier, "us")
             | (Model::MistralPixtralLarge2502V1, "us") => {

crates/debugger_ui/src/new_session_modal.rs

@@ -15,8 +15,9 @@ use dap::{
 use editor::{Editor, EditorElement, EditorStyle};
 use fuzzy::{StringMatch, StringMatchCandidate};
 use gpui::{
-    Animation, AnimationExt as _, App, AppContext, DismissEvent, Entity, EventEmitter, FocusHandle,
-    Focusable, Render, Subscription, TextStyle, Transformation, WeakEntity, percentage,
+    Action, Animation, AnimationExt as _, App, AppContext, DismissEvent, Entity, EventEmitter,
+    FocusHandle, Focusable, Render, Subscription, TextStyle, Transformation, WeakEntity,
+    percentage,
 };
 use picker::{Picker, PickerDelegate, highlighted_match_with_paths::HighlightedMatch};
 use project::{ProjectPath, TaskContexts, TaskSourceKind, task_store::TaskStore};
@@ -153,7 +154,7 @@ impl NewSessionModal {
                         attach_mode,
                         custom_mode,
                         debugger: None,
-                        mode: NewSessionMode::Launch,
+                        mode: NewSessionMode::Scenarios,
                         debug_panel: debug_panel.downgrade(),
                         workspace: workspace_handle,
                         save_scenario_state: None,
@@ -168,15 +169,15 @@ impl NewSessionModal {
     }
 
     fn render_mode(&mut self, window: &mut Window, cx: &mut Context<Self>) -> impl ui::IntoElement {
-        let dap_menu = self.adapter_drop_down_menu(window, cx);
+        let dap_menu = self.render_adapter_menu(window, cx);
         match self.mode {
             NewSessionMode::Attach => self.attach_mode.update(cx, |this, cx| {
                 this.clone().render(window, cx).into_any_element()
             }),
-            NewSessionMode::Custom => self.custom_mode.update(cx, |this, cx| {
+            NewSessionMode::Launch => self.custom_mode.update(cx, |this, cx| {
                 this.clone().render(dap_menu, window, cx).into_any_element()
             }),
-            NewSessionMode::Launch => v_flex()
+            NewSessionMode::Scenarios => v_flex()
                 .w(rems(34.))
                 .child(self.launch_picker.clone())
                 .into_any_element(),
@@ -186,14 +187,14 @@ impl NewSessionModal {
     fn mode_focus_handle(&self, cx: &App) -> FocusHandle {
         match self.mode {
             NewSessionMode::Attach => self.attach_mode.read(cx).attach_picker.focus_handle(cx),
-            NewSessionMode::Custom => self.custom_mode.read(cx).program.focus_handle(cx),
-            NewSessionMode::Launch => self.launch_picker.focus_handle(cx),
+            NewSessionMode::Launch => self.custom_mode.read(cx).program.focus_handle(cx),
+            NewSessionMode::Scenarios => self.launch_picker.focus_handle(cx),
         }
     }
 
     fn debug_scenario(&self, debugger: &str, cx: &App) -> Option<DebugScenario> {
         let request = match self.mode {
-            NewSessionMode::Custom => Some(DebugRequest::Launch(
+            NewSessionMode::Launch => Some(DebugRequest::Launch(
                 self.custom_mode.read(cx).debug_request(cx),
             )),
             NewSessionMode::Attach => Some(DebugRequest::Attach(
@@ -203,7 +204,7 @@ impl NewSessionModal {
         }?;
         let label = suggested_label(&request, debugger);
 
-        let stop_on_entry = if let NewSessionMode::Custom = &self.mode {
+        let stop_on_entry = if let NewSessionMode::Launch = &self.mode {
             Some(self.custom_mode.read(cx).stop_on_entry.selected())
         } else {
             None
@@ -226,7 +227,7 @@ impl NewSessionModal {
             return;
         };
 
-        if let NewSessionMode::Launch = &self.mode {
+        if let NewSessionMode::Scenarios = &self.mode {
             self.launch_picker.update(cx, |picker, cx| {
                 picker.delegate.confirm(false, window, cx);
             });
@@ -284,11 +285,28 @@ impl NewSessionModal {
         self.launch_picker.read(cx).delegate.task_contexts.clone()
     }
 
-    fn adapter_drop_down_menu(
+    fn render_adapter_menu(
         &mut self,
         window: &mut Window,
         cx: &mut Context<Self>,
     ) -> ui::DropdownMenu {
+        let mode = self.mode;
+        if let Some((_, scenario)) = self.launch_picker.update(cx, |picker, cx| {
+            picker
+                .delegate
+                .candidates
+                .get(picker.delegate.selected_index)
+                .filter(|_| mode == NewSessionMode::Scenarios)
+                .cloned()
+        }) {
+            return DropdownMenu::new(
+                "dap-adapter-disabled-picker",
+                scenario.adapter.clone(),
+                ContextMenu::build(window, cx, |menu, _, _| menu),
+            )
+            .disabled(true);
+        }
+
         let workspace = self.workspace.clone();
         let weak = cx.weak_entity();
         let active_buffer = self.task_contexts(cx).and_then(|tc| {
@@ -354,19 +372,19 @@ impl NewSessionModal {
 
 static SELECT_DEBUGGER_LABEL: SharedString = SharedString::new_static("Select Debugger");
 
-#[derive(Clone)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum NewSessionMode {
-    Custom,
-    Attach,
     Launch,
+    Attach,
+    Scenarios,
 }
 
 impl std::fmt::Display for NewSessionMode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let mode = match self {
-            NewSessionMode::Launch => "Launch".to_owned(),
+            NewSessionMode::Scenarios => "Scenarios".to_owned(),
             NewSessionMode::Attach => "Attach".to_owned(),
-            NewSessionMode::Custom => "Custom".to_owned(),
+            NewSessionMode::Launch => "Launch".to_owned(),
         };
 
         write!(f, "{}", mode)
@@ -434,11 +452,15 @@ impl Render for NewSessionModal {
             .on_action(cx.listener(|_, _: &menu::Cancel, _, cx| {
                 cx.emit(DismissEvent);
             }))
+            // .on_action(cx.listener(|this, _: &NewCustomSession, window, cx| {
+            //     this.mode = NewSessionMode::Launch;
+            //     this.mode_focus_handle(cx).focus(window);
+            // }))
             .on_action(
                 cx.listener(|this, _: &pane::ActivatePreviousItem, window, cx| {
                     this.mode = match this.mode {
-                        NewSessionMode::Attach => NewSessionMode::Launch,
-                        NewSessionMode::Launch => NewSessionMode::Attach,
+                        NewSessionMode::Attach => NewSessionMode::Scenarios,
+                        NewSessionMode::Scenarios => NewSessionMode::Attach,
                         _ => {
                             return;
                         }
@@ -449,8 +471,8 @@ impl Render for NewSessionModal {
             )
             .on_action(cx.listener(|this, _: &pane::ActivateNextItem, window, cx| {
                 this.mode = match this.mode {
-                    NewSessionMode::Attach => NewSessionMode::Launch,
-                    NewSessionMode::Launch => NewSessionMode::Attach,
+                    NewSessionMode::Attach => NewSessionMode::Scenarios,
+                    NewSessionMode::Scenarios => NewSessionMode::Attach,
                     _ => {
                         return;
                     }
@@ -468,19 +490,19 @@ impl Render for NewSessionModal {
                             .justify_start()
                             .w_full()
                             .child(
-                                ToggleButton::new("debugger-session-ui-picker-button", "Launch")
+                                ToggleButton::new("debugger-session-ui-picker-button", NewSessionMode::Scenarios.to_string())
                                     .size(ButtonSize::Default)
                                     .style(ui::ButtonStyle::Subtle)
-                                    .toggle_state(matches!(self.mode, NewSessionMode::Launch))
+                                    .toggle_state(matches!(self.mode, NewSessionMode::Scenarios))
                                     .on_click(cx.listener(|this, _, window, cx| {
-                                        this.mode = NewSessionMode::Launch;
+                                        this.mode = NewSessionMode::Scenarios;
                                         this.mode_focus_handle(cx).focus(window);
                                         cx.notify();
                                     }))
                                     .first(),
                             )
                             .child(
-                                ToggleButton::new("debugger-session-ui-attach-button", "Attach")
+                                ToggleButton::new("debugger-session-ui-attach-button", NewSessionMode::Attach.to_string())
                                     .size(ButtonSize::Default)
                                     .toggle_state(matches!(self.mode, NewSessionMode::Attach))
                                     .style(ui::ButtonStyle::Subtle)
@@ -499,217 +521,233 @@ impl Render for NewSessionModal {
                                         cx.notify();
                                     }))
                                     .last(),
-                            ),
+                            )
+                            .child(
+                                ToggleButton::new("debugger-session-ui-launch-button", NewSessionMode::Launch.to_string())
+                                    .size(ButtonSize::Default)
+                                    .style(ui::ButtonStyle::Subtle)
+                                    .toggle_state(matches!(self.mode, NewSessionMode::Launch))
+                                    .on_click(cx.listener(|this, _, window, cx| {
+                                        this.mode = NewSessionMode::Launch;
+                                        this.mode_focus_handle(cx).focus(window);
+                                        cx.notify();
+                                    }))
+                                    .first(),
+                            )
                     )
-                    .justify_between()
+                    .justify_end()
+                    .child(self.render_adapter_menu(window, cx))
                     .border_color(cx.theme().colors().border_variant)
                     .border_b_1(),
             )
             .child(v_flex().child(self.render_mode(window, cx)))
-            .child(
-                h_flex()
-                    .justify_between()
-                    .gap_2()
-                    .p_2()
-                    .border_color(cx.theme().colors().border_variant)
-                    .border_t_1()
-                    .w_full()
-                    .child(match self.mode {
-                        NewSessionMode::Attach => {
-                            div().child(self.adapter_drop_down_menu(window, cx))
-                        }
-                        NewSessionMode::Launch => div().child(
-                            Button::new("new-session-modal-custom", "Custom").on_click({
-                                let this = cx.weak_entity();
-                                move |_, window, cx| {
-                                    this.update(cx, |this, cx| {
-                                        this.mode = NewSessionMode::Custom;
-                                        this.mode_focus_handle(cx).focus(window);
-                                    })
-                                    .ok();
-                                }
-                            }),
-                        ),
-                        NewSessionMode::Custom => h_flex()
-                            .child(
-                                Button::new("new-session-modal-back", "Save to .zed/debug.json...")
-                                    .on_click(cx.listener(|this, _, window, cx| {
-                                        let Some(save_scenario) = this
-                                            .debugger
-                                            .as_ref()
-                                            .and_then(|debugger| this.debug_scenario(&debugger, cx))
-                                            .zip(
-                                                this.task_contexts(cx)
-                                                    .and_then(|tcx| tcx.worktree()),
-                                            )
-                                            .and_then(|(scenario, worktree_id)| {
-                                                this.debug_panel
-                                                    .update(cx, |panel, cx| {
-                                                        panel.save_scenario(
-                                                            &scenario,
-                                                            worktree_id,
-                                                            window,
-                                                            cx,
-                                                        )
-                                                    })
-                                                    .ok()
-                                            })
-                                        else {
-                                            return;
-                                        };
-
-                                        this.save_scenario_state = Some(SaveScenarioState::Saving);
-
-                                        cx.spawn(async move |this, cx| {
-                                            let res = save_scenario.await;
-
-                                            this.update(cx, |this, _| match res {
-                                                Ok(saved_file) => {
-                                                    this.save_scenario_state =
-                                                        Some(SaveScenarioState::Saved(saved_file))
-                                                }
-                                                Err(error) => {
-                                                    this.save_scenario_state =
-                                                        Some(SaveScenarioState::Failed(
-                                                            error.to_string().into(),
-                                                        ))
-                                                }
-                                            })
-                                            .ok();
-
-                                            cx.background_executor()
-                                                .timer(Duration::from_secs(2))
-                                                .await;
-                                            this.update(cx, |this, _| {
-                                                this.save_scenario_state.take()
-                                            })
-                                            .ok();
+            .when(self.mode == NewSessionMode::Launch, |el|{
+                // FIXME clean up
+                el.child(
+                    h_flex()
+                        .justify_between()
+                        .gap_2()
+                        .p_2()
+                        .border_color(cx.theme().colors().border_variant)
+                        .border_t_1()
+                        .w_full()
+                        .child(match self.mode {
+                            NewSessionMode::Attach => {
+                                div().child(self.render_adapter_menu(window, cx))
+                            }
+                            NewSessionMode::Scenarios => div().child(
+                                Button::new("new-session-modal-custom", "Custom").on_click({
+                                    let this = cx.weak_entity();
+                                    move |_, window, cx| {
+                                        this.update(cx, |this, cx| {
+                                            this.mode = NewSessionMode::Launch;
+                                            this.mode_focus_handle(cx).focus(window);
                                         })
-                                        .detach();
-                                    }))
-                                    .disabled(
+                                        .ok();
+                                    }
+                                }),
+                            ),
+                            NewSessionMode::Launch => h_flex()
+                                .child(
+                                    Button::new("new-session-modal-back", "Save to .zed/debug.json...")
+                                        .on_click(cx.listener(|this, _, window, cx| {
+                                            let Some(save_scenario) = this
+                                                .debugger
+                                                .as_ref()
+                                                .and_then(|debugger| this.debug_scenario(&debugger, cx))
+                                                .zip(
+                                                    this.task_contexts(cx)
+                                                        .and_then(|tcx| tcx.worktree()),
+                                                )
+                                                .and_then(|(scenario, worktree_id)| {
+                                                    this.debug_panel
+                                                        .update(cx, |panel, cx| {
+                                                            panel.save_scenario(
+                                                                &scenario,
+                                                                worktree_id,
+                                                                window,
+                                                                cx,
+                                                            )
+                                                        })
+                                                        .ok()
+                                                })
+                                            else {
+                                                return;
+                                            };
+
+                                            this.save_scenario_state = Some(SaveScenarioState::Saving);
+
+                                            cx.spawn(async move |this, cx| {
+                                                let res = save_scenario.await;
+
+                                                this.update(cx, |this, _| match res {
+                                                    Ok(saved_file) => {
+                                                        this.save_scenario_state =
+                                                            Some(SaveScenarioState::Saved(saved_file))
+                                                    }
+                                                    Err(error) => {
+                                                        this.save_scenario_state =
+                                                            Some(SaveScenarioState::Failed(
+                                                                error.to_string().into(),
+                                                            ))
+                                                    }
+                                                })
+                                                .ok();
+
+                                                cx.background_executor()
+                                                    .timer(Duration::from_secs(2))
+                                                    .await;
+                                                this.update(cx, |this, _| {
+                                                    this.save_scenario_state.take()
+                                                })
+                                                .ok();
+                                            })
+                                            .detach();
+                                        }))
+                                        .disabled(
+                                            self.debugger.is_none()
+                                                || self
+                                                    .custom_mode
+                                                    .read(cx)
+                                                    .program
+                                                    .read(cx)
+                                                    .is_empty(cx)
+                                                || self.save_scenario_state.is_some(),
+                                        ),
+                                )
+                                .when_some(self.save_scenario_state.as_ref(), {
+                                    let this_entity = this.clone();
+
+                                    move |this, save_state| match save_state {
+                                        SaveScenarioState::Saved(saved_path) => this.child(
+                                            IconButton::new(
+                                                "new-session-modal-go-to-file",
+                                                IconName::ArrowUpRight,
+                                            )
+                                            .icon_size(IconSize::Small)
+                                            .icon_color(Color::Muted)
+                                            .on_click({
+                                                let this_entity = this_entity.clone();
+                                                let saved_path = saved_path.clone();
+                                                move |_, window, cx| {
+                                                    window
+                                                        .spawn(cx, {
+                                                            let this_entity = this_entity.clone();
+                                                            let saved_path = saved_path.clone();
+
+                                                            async move |cx| {
+                                                                this_entity
+                                                                    .update_in(
+                                                                        cx,
+                                                                        |this, window, cx| {
+                                                                            this.workspace.update(
+                                                                                cx,
+                                                                                |workspace, cx| {
+                                                                                    workspace.open_path(
+                                                                                        saved_path
+                                                                                            .clone(),
+                                                                                        None,
+                                                                                        true,
+                                                                                        window,
+                                                                                        cx,
+                                                                                    )
+                                                                                },
+                                                                            )
+                                                                        },
+                                                                    )??
+                                                                    .await?;
+
+                                                                this_entity
+                                                                    .update(cx, |_, cx| {
+                                                                        cx.emit(DismissEvent)
+                                                                    })
+                                                                    .ok();
+
+                                                                anyhow::Ok(())
+                                                            }
+                                                        })
+                                                        .detach();
+                                                }
+                                            }),
+                                        ),
+                                        SaveScenarioState::Saving => this.child(
+                                            Icon::new(IconName::Spinner)
+                                                .size(IconSize::Small)
+                                                .color(Color::Muted)
+                                                .with_animation(
+                                                    "Spinner",
+                                                    Animation::new(Duration::from_secs(3)).repeat(),
+                                                    |icon, delta| {
+                                                        icon.transform(Transformation::rotate(
+                                                            percentage(delta),
+                                                        ))
+                                                    },
+                                                ),
+                                        ),
+                                        SaveScenarioState::Failed(error_msg) => this.child(
+                                            IconButton::new("Failed Scenario Saved", IconName::X)
+                                                .icon_size(IconSize::Small)
+                                                .icon_color(Color::Error)
+                                                .tooltip(ui::Tooltip::text(error_msg.clone())),
+                                        ),
+                                    }
+                                }),
+                        })
+                        .child(
+                            Button::new("debugger-spawn", "Start")
+                                .on_click(cx.listener(|this, _, window, cx| match &this.mode {
+                                    NewSessionMode::Scenarios => {
+                                        this.launch_picker.update(cx, |picker, cx| {
+                                            picker.delegate.confirm(true, window, cx)
+                                        })
+                                    }
+                                    _ => this.start_new_session(window, cx),
+                                }))
+                                .disabled(match self.mode {
+                                    NewSessionMode::Scenarios => {
+                                        !self.launch_picker.read(cx).delegate.matches.is_empty()
+                                    }
+                                    NewSessionMode::Attach => {
                                         self.debugger.is_none()
                                             || self
-                                                .custom_mode
+                                                .attach_mode
                                                 .read(cx)
-                                                .program
+                                                .attach_picker
                                                 .read(cx)
-                                                .is_empty(cx)
-                                            || self.save_scenario_state.is_some(),
-                                    ),
-                            )
-                            .when_some(self.save_scenario_state.as_ref(), {
-                                let this_entity = this.clone();
-
-                                move |this, save_state| match save_state {
-                                    SaveScenarioState::Saved(saved_path) => this.child(
-                                        IconButton::new(
-                                            "new-session-modal-go-to-file",
-                                            IconName::ArrowUpRight,
-                                        )
-                                        .icon_size(IconSize::Small)
-                                        .icon_color(Color::Muted)
-                                        .on_click({
-                                            let this_entity = this_entity.clone();
-                                            let saved_path = saved_path.clone();
-                                            move |_, window, cx| {
-                                                window
-                                                    .spawn(cx, {
-                                                        let this_entity = this_entity.clone();
-                                                        let saved_path = saved_path.clone();
-
-                                                        async move |cx| {
-                                                            this_entity
-                                                                .update_in(
-                                                                    cx,
-                                                                    |this, window, cx| {
-                                                                        this.workspace.update(
-                                                                            cx,
-                                                                            |workspace, cx| {
-                                                                                workspace.open_path(
-                                                                                    saved_path
-                                                                                        .clone(),
-                                                                                    None,
-                                                                                    true,
-                                                                                    window,
-                                                                                    cx,
-                                                                                )
-                                                                            },
-                                                                        )
-                                                                    },
-                                                                )??
-                                                                .await?;
-
-                                                            this_entity
-                                                                .update(cx, |_, cx| {
-                                                                    cx.emit(DismissEvent)
-                                                                })
-                                                                .ok();
-
-                                                            anyhow::Ok(())
-                                                        }
-                                                    })
-                                                    .detach();
-                                            }
-                                        }),
-                                    ),
-                                    SaveScenarioState::Saving => this.child(
-                                        Icon::new(IconName::Spinner)
-                                            .size(IconSize::Small)
-                                            .color(Color::Muted)
-                                            .with_animation(
-                                                "Spinner",
-                                                Animation::new(Duration::from_secs(3)).repeat(),
-                                                |icon, delta| {
-                                                    icon.transform(Transformation::rotate(
-                                                        percentage(delta),
-                                                    ))
-                                                },
-                                            ),
-                                    ),
-                                    SaveScenarioState::Failed(error_msg) => this.child(
-                                        IconButton::new("Failed Scenario Saved", IconName::X)
-                                            .icon_size(IconSize::Small)
-                                            .icon_color(Color::Error)
-                                            .tooltip(ui::Tooltip::text(error_msg.clone())),
-                                    ),
-                                }
-                            }),
-                    })
-                    .child(
-                        Button::new("debugger-spawn", "Start")
-                            .on_click(cx.listener(|this, _, window, cx| match &this.mode {
-                                NewSessionMode::Launch => {
-                                    this.launch_picker.update(cx, |picker, cx| {
-                                        picker.delegate.confirm(true, window, cx)
-                                    })
-                                }
-                                _ => this.start_new_session(window, cx),
-                            }))
-                            .disabled(match self.mode {
-                                NewSessionMode::Launch => {
-                                    !self.launch_picker.read(cx).delegate.matches.is_empty()
-                                }
-                                NewSessionMode::Attach => {
-                                    self.debugger.is_none()
-                                        || self
-                                            .attach_mode
-                                            .read(cx)
-                                            .attach_picker
-                                            .read(cx)
-                                            .picker
-                                            .read(cx)
-                                            .delegate
-                                            .match_count()
-                                            == 0
-                                }
-                                NewSessionMode::Custom => {
-                                    self.debugger.is_none()
-                                        || self.custom_mode.read(cx).program.read(cx).is_empty(cx)
-                                }
-                            }),
-                    ),
-            )
+                                                .picker
+                                                .read(cx)
+                                                .delegate
+                                                .match_count()
+                                                == 0
+                                    }
+                                    NewSessionMode::Launch => {
+                                        self.debugger.is_none()
+                                            || self.custom_mode.read(cx).program.read(cx).is_empty(cx)
+                                    }
+                                }),
+                        ),
+                )
+            })
     }
 }
 
@@ -825,16 +863,16 @@ impl CustomMode {
             .w_full()
             .gap_3()
             .track_focus(&self.program.focus_handle(cx))
-            .child(
-                h_flex()
-                    .child(
-                        Label::new("Debugger")
-                            .size(ui::LabelSize::Small)
-                            .color(Color::Muted),
-                    )
-                    .gap(ui::DynamicSpacing::Base08.rems(cx))
-                    .child(adapter_menu),
-            )
+            // .child(
+            //     h_flex()
+            //         .child(
+            //             Label::new("Debugger")
+            //                 .size(ui::LabelSize::Small)
+            //                 .color(Color::Muted),
+            //         )
+            //         .gap(ui::DynamicSpacing::Base08.rems(cx))
+            //         .child(adapter_menu),
+            // )
             .child(render_editor(&self.program, window, cx))
             .child(render_editor(&self.cwd, window, cx))
             .child(
@@ -1112,6 +1150,19 @@ impl PickerDelegate for DebugScenarioDelegate {
         window: &mut Window,
         cx: &mut Context<picker::Picker<Self>>,
     ) -> Option<Self::ListItem> {
+        // if ix == self.matches.len() {
+        //     return Some(
+        //         ListItem::new("new-session-modal-custom")
+        //             .child(Label::new("Custom"))
+        //             .inset(true)
+        //             .spacing(ListItemSpacing::Sparse)
+        //             .toggle_state(selected)
+        //             .on_click(move |_, window, cx| {
+        //                 window.dispatch_action(NewCustomSession.boxed_clone(), cx);
+        //             }),
+        //     );
+        // }
+
         let hit = &self.matches[ix];
 
         let highlighted_location = HighlightedMatch {

crates/debugger_ui/src/session/running.rs

@@ -927,6 +927,7 @@ impl RunningState {
                     .ok_or_else(|| anyhow!("{}: is not a valid adapter name", &adapter))
                     .map(|adapter| adapter.config_from_zed_format(zed_config))??;
                 config = scenario.config;
+                Self::substitute_variables_in_config(&mut config, &task_context);
             } else {
                 anyhow::bail!("No request or build provided");
             };

crates/eval/src/eval.rs

@@ -34,7 +34,7 @@ use std::env;
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::sync::{Arc, LazyLock};
-use util::{ConnectionResult, ResultExt as _};
+use util::ResultExt as _;
 
 static CARGO_MANIFEST_DIR: LazyLock<PathBuf> =
     LazyLock::new(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")));
@@ -144,13 +144,6 @@ fn main() {
         cx.spawn(async move |cx| {
             authenticate_task.await.unwrap();
 
-            match app_state.client.authenticate_and_connect(true, cx).await {
-                ConnectionResult::Timeout => panic!("Timeout while trying to authenticate_and_connect"),
-                ConnectionResult::ConnectionReset => panic!("Connection reset while trying to authenticate_and_connect"),
-                ConnectionResult::Result(Err(err)) => panic!("Error while trying to authenticate_and_connect: {err:?}"),
-                ConnectionResult::Result(Ok(())) => {}
-            }
-
             let mut examples = Vec::new();
 
             const COLORS: [&str; 12] = [
@@ -439,6 +432,7 @@ pub fn init(cx: &mut App) -> Arc<AgentAppState> {
         client.clone(),
         prompt_builder.clone(),
         languages.clone(),
+        true,
         cx,
     );
     assistant_tools::init(client.http_client(), cx);

crates/eval/src/example.rs

@@ -231,6 +231,10 @@ impl ExampleContext {
                     Ok(StopReason::MaxTokens) => {
                         tx.try_send(Err(anyhow!("Exceeded maximum tokens"))).ok();
                     }
+                    Ok(StopReason::Refusal) => {
+                        tx.try_send(Err(anyhow!("Model refused to generate content")))
+                            .ok();
+                    }
                     Err(err) => {
                         tx.try_send(Err(anyhow!(err.clone()))).ok();
                     }

crates/language_model/src/language_model.rs

@@ -100,6 +100,7 @@ pub enum StopReason {
     EndTurn,
     MaxTokens,
     ToolUse,
+    Refusal,
 }
 
 #[derive(Debug, Clone, Copy)]

crates/language_models/src/provider/anthropic.rs

@@ -240,8 +240,8 @@ impl LanguageModelProvider for AnthropicLanguageModelProvider {
 
     fn recommended_models(&self, _cx: &App) -> Vec<Arc<dyn LanguageModel>> {
         [
-            anthropic::Model::Claude3_7Sonnet,
-            anthropic::Model::Claude3_7SonnetThinking,
+            anthropic::Model::ClaudeSonnet4,
+            anthropic::Model::ClaudeSonnet4Thinking,
         ]
         .into_iter()
         .map(|model| self.create_language_model(model))
@@ -825,6 +825,7 @@ impl AnthropicEventMapper {
                         "end_turn" => StopReason::EndTurn,
                         "max_tokens" => StopReason::MaxTokens,
                         "tool_use" => StopReason::ToolUse,
+                        "refusal" => StopReason::Refusal,
                         _ => {
                             log::error!("Unexpected anthropic stop_reason: {stop_reason}");
                             StopReason::EndTurn

crates/language_models/src/provider/cloud.rs

@@ -278,7 +278,7 @@ impl LanguageModelProvider for CloudLanguageModelProvider {
 
     fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
         let llm_api_token = self.state.read(cx).llm_api_token.clone();
-        let model = CloudModel::Anthropic(anthropic::Model::Claude3_7Sonnet);
+        let model = CloudModel::Anthropic(anthropic::Model::ClaudeSonnet4);
         Some(self.create_language_model(model, llm_api_token))
     }
 
@@ -291,8 +291,8 @@ impl LanguageModelProvider for CloudLanguageModelProvider {
     fn recommended_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
         let llm_api_token = self.state.read(cx).llm_api_token.clone();
         [
-            CloudModel::Anthropic(anthropic::Model::Claude3_7Sonnet),
-            CloudModel::Anthropic(anthropic::Model::Claude3_7SonnetThinking),
+            CloudModel::Anthropic(anthropic::Model::ClaudeSonnet4),
+            CloudModel::Anthropic(anthropic::Model::ClaudeSonnet4Thinking),
         ]
         .into_iter()
         .map(|model| self.create_language_model(model, llm_api_token.clone()))

crates/zed/src/main.rs

@@ -519,6 +519,7 @@ fn main() {
             app_state.client.clone(),
             prompt_builder.clone(),
             app_state.languages.clone(),
+            false,
             cx,
         );
         assistant_tools::init(app_state.client.http_client(), cx);

crates/zed/src/zed.rs

@@ -4295,6 +4295,7 @@ mod tests {
                 app_state.client.clone(),
                 prompt_builder.clone(),
                 app_state.languages.clone(),
+                false,
                 cx,
             );
             repl::init(app_state.fs.clone(), cx);

docs/src/ai/models.md

@@ -7,6 +7,8 @@ Zed’s plans offer hosted versions of major LLM’s, generally with higher rate
 | Claude 3.5 Sonnet | Anthropic | ❌       | 60k            | $0.04            | N/A               |
 | Claude 3.7 Sonnet | Anthropic | ❌       | 120k           | $0.04            | N/A               |
 | Claude 3.7 Sonnet | Anthropic | ✅       | 200k           | N/A              | $0.05             |
+| Claude Sonnet 4   | Anthropic | ❌       | 120k           | $0.04            | N/A               |
+| Claude Sonnet 4   | Anthropic | ✅       | 200k           | N/A              | $0.05             |
 
 ## Usage {#usage}