Add callable workflow for extension repositories (#43082)

This starts the work on a workflow that can be invoked in extension CI
to test changes on extension repositories.

Release Notes:

- N/A

---------

Co-authored-by: Agus Zubiaga <agus@zed.dev>
Co-authored-by: Conrad Irwin <conrad.irwin@gmail.com>

tooling/xtask/src/tasks/workflows.rs

@@ -7,6 +7,7 @@ mod after_release;
 mod cherry_pick;
 mod compare_perf;
 mod danger;
+mod extension_tests;
 mod nix_build;
 mod release_nightly;
 mod run_bundling;
@@ -39,6 +40,7 @@ pub fn run_workflows(_: GenerateWorkflowArgs) -> Result<()> {
         ),
         ("run_agent_evals.yml", run_agent_evals::run_agent_evals()),
         ("after_release.yml", after_release::after_release()),
+        ("extension_tests.yml", extension_tests::extension_tests()),
     ];
     fs::create_dir_all(dir)
         .with_context(|| format!("Failed to create directory: {}", dir.display()))?;

tooling/xtask/src/tasks/workflows/after_release.rs

@@ -3,7 +3,7 @@ use gh_workflow::*;
 use crate::tasks::workflows::{
     release::{self, notify_on_failure},
     runners,
-    steps::{NamedJob, checkout_repo, dependant_job, named},
+    steps::{CommonJobConditions, NamedJob, checkout_repo, dependant_job, named},
     vars::{self, StepOutput},
 };
 
@@ -43,9 +43,7 @@ fn rebuild_releases_page() -> NamedJob {
     named::job(
         Job::default()
             .runs_on(runners::LINUX_SMALL)
-            .cond(Expression::new(
-                "github.repository_owner == 'zed-industries'",
-            ))
+            .with_repository_owner_guard()
             .add_step(refresh_cloud_releases())
             .add_step(redeploy_zed_dev()),
     )
@@ -95,9 +93,7 @@ fn post_to_discord(deps: &[&NamedJob]) -> NamedJob {
     }
     let job = dependant_job(deps)
         .runs_on(runners::LINUX_SMALL)
-        .cond(Expression::new(
-            "github.repository_owner == 'zed-industries'",
-        ))
+        .with_repository_owner_guard()
         .add_step(get_release_url())
         .add_step(get_content())
         .add_step(discord_webhook_action());
@@ -145,9 +141,7 @@ fn publish_winget() -> NamedJob {
 fn create_sentry_release() -> NamedJob {
     let job = Job::default()
         .runs_on(runners::LINUX_SMALL)
-        .cond(Expression::new(
-            "github.repository_owner == 'zed-industries'",
-        ))
+        .with_repository_owner_guard()
         .add_step(checkout_repo())
         .add_step(release::create_sentry_release());
     named::job(job)

tooling/xtask/src/tasks/workflows/danger.rs

@@ -1,6 +1,6 @@
 use gh_workflow::*;
 
-use crate::tasks::workflows::steps::{NamedJob, named};
+use crate::tasks::workflows::steps::{CommonJobConditions, NamedJob, named};
 
 use super::{runners, steps};
 
@@ -42,9 +42,7 @@ fn danger_job() -> NamedJob {
     NamedJob {
         name: "danger".to_string(),
         job: Job::default()
-            .cond(Expression::new(
-                "github.repository_owner == 'zed-industries'",
-            ))
+            .with_repository_owner_guard()
             .runs_on(runners::LINUX_SMALL)
             .add_step(steps::checkout_repo())
             .add_step(steps::setup_pnpm())

tooling/xtask/src/tasks/workflows/extension_tests.rs

@@ -0,0 +1,129 @@
+use gh_workflow::*;
+use indoc::indoc;
+
+use crate::tasks::workflows::{
+    run_tests::{orchestrate, tests_pass},
+    runners,
+    steps::{self, CommonJobConditions, FluentBuilder, NamedJob, named},
+    vars::{PathCondition, StepOutput, one_workflow_per_non_main_branch},
+};
+
+const RUN_TESTS_INPUT: &str = "run_tests";
+const ZED_EXTENSION_CLI_SHA: &str = "7cfce605704d41ca247e3f84804bf323f6c6caaf";
+
+// This is used by various extensions repos in the zed-extensions org to run automated tests.
+pub(crate) fn extension_tests() -> Workflow {
+    let should_check_rust = PathCondition::new("check_rust", r"^(Cargo.lock|Cargo.toml|.*\.rs)$");
+    let should_check_extension = PathCondition::new("check_extension", r"^.*\.scm$");
+
+    let orchestrate = orchestrate(&[&should_check_rust, &should_check_extension]);
+
+    let jobs = [
+        orchestrate,
+        should_check_rust.guard(check_rust()),
+        should_check_extension.guard(check_extension()),
+    ];
+
+    let tests_pass = tests_pass(&jobs);
+
+    named::workflow()
+        .add_event(
+            Event::default().workflow_call(WorkflowCall::default().add_input(
+                RUN_TESTS_INPUT,
+                WorkflowCallInput {
+                    description: "Whether the workflow should run rust tests".into(),
+                    required: true,
+                    input_type: "boolean".into(),
+                    default: None,
+                },
+            )),
+        )
+        .concurrency(one_workflow_per_non_main_branch())
+        .add_env(("CARGO_TERM_COLOR", "always"))
+        .add_env(("RUST_BACKTRACE", 1))
+        .add_env(("CARGO_INCREMENTAL", 0))
+        .add_env(("ZED_EXTENSION_CLI_SHA", ZED_EXTENSION_CLI_SHA))
+        .map(|workflow| {
+            jobs.into_iter()
+                .chain([tests_pass])
+                .fold(workflow, |workflow, job| {
+                    workflow.add_job(job.name, job.job)
+                })
+        })
+}
+
+fn run_clippy() -> Step<Run> {
+    named::bash("cargo clippy --release --all-targets --all-features -- --deny warnings")
+}
+
+fn check_rust() -> NamedJob {
+    let job = Job::default()
+        .with_repository_owner_guard()
+        .runs_on(runners::LINUX_DEFAULT)
+        .timeout_minutes(3u32)
+        .add_step(steps::checkout_repo())
+        .add_step(steps::cache_rust_dependencies_namespace())
+        .add_step(steps::cargo_fmt())
+        .add_step(run_clippy())
+        .add_step(
+            steps::cargo_install_nextest()
+                .if_condition(Expression::new(format!("inputs.{RUN_TESTS_INPUT}"))),
+        )
+        .add_step(
+            steps::cargo_nextest(runners::Platform::Linux)
+                .if_condition(Expression::new(format!("inputs.{RUN_TESTS_INPUT}"))),
+        );
+
+    named::job(job)
+}
+
+fn check_extension() -> NamedJob {
+    let (cache_download, cache_hit) = cache_zed_extension_cli();
+    let job = Job::default()
+        .with_repository_owner_guard()
+        .runs_on(runners::LINUX_SMALL)
+        .timeout_minutes(1u32)
+        .add_step(steps::checkout_repo())
+        .add_step(cache_download)
+        .add_step(download_zed_extension_cli(cache_hit))
+        .add_step(check());
+
+    named::job(job)
+}
+
+pub fn cache_zed_extension_cli() -> (Step<Use>, StepOutput) {
+    let step = named::uses(
+        "actions",
+        "cache",
+        "0057852bfaa89a56745cba8c7296529d2fc39830",
+    )
+    .id("cache-zed-extension-cli")
+    .with(
+        Input::default()
+            .add("path", "zed-extension")
+            .add("key", "zed-extension-${{ env.ZED_EXTENSION_CLI_SHA }}"),
+    );
+    let output = StepOutput::new(&step, "cache-hit");
+    (step, output)
+}
+
+pub fn download_zed_extension_cli(cache_hit: StepOutput) -> Step<Run> {
+    named::bash(
+    indoc! {
+        r#"
+        wget --quiet "https://zed-extension-cli.nyc3.digitaloceanspaces.com/$ZED_EXTENSION_CLI_SHA/x86_64-unknown-linux-gnu/zed-extension"
+        chmod +x zed-extension
+        "#,
+    }
+    ).if_condition(Expression::new(format!("{} != 'true'", cache_hit.expr())))
+}
+
+pub fn check() -> Step<Run> {
+    named::bash(indoc! {
+        r#"
+        mkdir -p /tmp/ext-scratch
+        mkdir -p /tmp/ext-output
+        ./zed-extension --source-dir . --scratch-dir /tmp/ext-scratch --output-dir /tmp/ext-output
+        "#
+    })
+}

tooling/xtask/src/tasks/workflows/nix_build.rs

@@ -1,6 +1,6 @@
 use crate::tasks::workflows::{
     runners::{Arch, Platform},
-    steps::NamedJob,
+    steps::{CommonJobConditions, NamedJob},
 };
 
 use super::{runners, steps, steps::named, vars};
@@ -71,9 +71,7 @@ pub(crate) fn build_nix(
     let mut job = Job::default()
         .timeout_minutes(60u32)
         .continue_on_error(true)
-        .cond(Expression::new(
-            "github.repository_owner == 'zed-industries'",
-        ))
+        .with_repository_owner_guard()
         .runs_on(runner)
         .add_env(("ZED_CLIENT_CHECKSUM_SEED", vars::ZED_CLIENT_CHECKSUM_SEED))
         .add_env(("ZED_MINIDUMP_ENDPOINT", vars::ZED_SENTRY_MINIDUMP_ENDPOINT))

tooling/xtask/src/tasks/workflows/release_nightly.rs

@@ -7,7 +7,7 @@ use crate::tasks::workflows::{
     run_bundling::{bundle_linux, bundle_mac, bundle_windows},
     run_tests::run_platform_tests,
     runners::{Arch, Platform, ReleaseChannel},
-    steps::{FluentBuilder, NamedJob},
+    steps::{CommonJobConditions, FluentBuilder, NamedJob},
 };
 
 use super::{runners, steps, steps::named, vars};
@@ -83,9 +83,7 @@ fn check_style() -> NamedJob {
 
 fn release_job(deps: &[&NamedJob]) -> Job {
     let job = Job::default()
-        .cond(Expression::new(
-            "github.repository_owner == 'zed-industries'",
-        ))
+        .with_repository_owner_guard()
         .timeout_minutes(60u32);
     if deps.len() > 0 {
         job.needs(deps.iter().map(|j| j.name.clone()).collect::<Vec<_>>())

tooling/xtask/src/tasks/workflows/run_tests.rs

@@ -4,7 +4,10 @@ use gh_workflow::{
 use indexmap::IndexMap;
 
 use crate::tasks::workflows::{
-    nix_build::build_nix, runners::Arch, steps::BASH_SHELL, vars::PathCondition,
+    nix_build::build_nix,
+    runners::Arch,
+    steps::{BASH_SHELL, CommonJobConditions, repository_owner_guard_expression},
+    vars::PathCondition,
 };
 
 use super::{
@@ -107,7 +110,7 @@ pub(crate) fn run_tests() -> Workflow {
 
 // Generates a bash script that checks changed files against regex patterns
 // and sets GitHub output variables accordingly
-fn orchestrate(rules: &[&PathCondition]) -> NamedJob {
+pub fn orchestrate(rules: &[&PathCondition]) -> NamedJob {
     let name = "orchestrate".to_owned();
     let step_name = "filter".to_owned();
     let mut script = String::new();
@@ -162,9 +165,7 @@ fn orchestrate(rules: &[&PathCondition]) -> NamedJob {
 
     let job = Job::default()
         .runs_on(runners::LINUX_SMALL)
-        .cond(Expression::new(
-            "github.repository_owner == 'zed-industries'",
-        ))
+        .with_repository_owner_guard()
         .outputs(outputs)
         .add_step(steps::checkout_repo().add_with((
             "fetch-depth",
@@ -180,7 +181,7 @@ fn orchestrate(rules: &[&PathCondition]) -> NamedJob {
     NamedJob { name, job }
 }
 
-pub(crate) fn tests_pass(jobs: &[NamedJob]) -> NamedJob {
+pub fn tests_pass(jobs: &[NamedJob]) -> NamedJob {
     let mut script = String::from(indoc::indoc! {r#"
         set +x
         EXIT_CODE=0
@@ -214,9 +215,7 @@ pub(crate) fn tests_pass(jobs: &[NamedJob]) -> NamedJob {
                 .map(|j| j.name.to_string())
                 .collect::<Vec<String>>(),
         )
-        .cond(Expression::new(
-            "github.repository_owner == 'zed-industries' && always()",
-        ))
+        .cond(repository_owner_guard_expression(true))
         .add_step(named::bash(&script));
 
     named::job(job)

tooling/xtask/src/tasks/workflows/steps.rs

@@ -94,18 +94,18 @@ pub fn clear_target_dir_if_large(platform: Platform) -> Step<Run> {
     }
 }
 
-pub(crate) fn clippy(platform: Platform) -> Step<Run> {
+pub fn clippy(platform: Platform) -> Step<Run> {
     match platform {
         Platform::Windows => named::pwsh("./script/clippy.ps1"),
         _ => named::bash("./script/clippy"),
     }
 }
 
-pub(crate) fn cache_rust_dependencies_namespace() -> Step<Use> {
+pub fn cache_rust_dependencies_namespace() -> Step<Use> {
     named::uses("namespacelabs", "nscloud-cache-action", "v1").add_with(("cache", "rust"))
 }
 
-fn setup_linux() -> Step<Run> {
+pub fn setup_linux() -> Step<Run> {
     named::bash("./script/linux")
 }
 
@@ -131,7 +131,7 @@ pub fn script(name: &str) -> Step<Run> {
     }
 }
 
-pub(crate) struct NamedJob {
+pub struct NamedJob {
     pub name: String,
     pub job: Job,
 }
@@ -145,11 +145,26 @@ pub(crate) struct NamedJob {
 //     }
 // }
 
+pub fn repository_owner_guard_expression(trigger_always: bool) -> Expression {
+    Expression::new(format!(
+        "(github.repository_owner == 'zed-industries' || github.repository_owner == 'zed-extensions'){}",
+        trigger_always.then_some(" && always()").unwrap_or_default()
+    ))
+}
+
+pub trait CommonJobConditions: Sized {
+    fn with_repository_owner_guard(self) -> Self;
+}
+
+impl CommonJobConditions for Job {
+    fn with_repository_owner_guard(self) -> Self {
+        self.cond(repository_owner_guard_expression(false))
+    }
+}
+
 pub(crate) fn release_job(deps: &[&NamedJob]) -> Job {
     dependant_job(deps)
-        .cond(Expression::new(
-            "github.repository_owner == 'zed-industries'",
-        ))
+        .with_repository_owner_guard()
         .timeout_minutes(60u32)
 }
 
@@ -169,7 +184,7 @@ impl FluentBuilder for Workflow {}
 /// Copied from GPUI to avoid adding GPUI as dependency
 /// todo(ci) just put this in gh-workflow
 #[allow(unused)]
-pub(crate) trait FluentBuilder {
+pub trait FluentBuilder {
     /// Imperatively modify self with the given closure.
     fn map<U>(self, f: impl FnOnce(Self) -> U) -> U
     where
@@ -223,34 +238,34 @@ pub(crate) trait FluentBuilder {
 
 // (janky) helper to generate steps with a name that corresponds
 // to the name of the calling function.
-pub(crate) mod named {
+pub mod named {
     use super::*;
 
     /// Returns a uses step with the same name as the enclosing function.
     /// (You shouldn't inline this function into the workflow definition, you must
     /// wrap it in a new function.)
-    pub(crate) fn uses(owner: &str, repo: &str, ref_: &str) -> Step<Use> {
+    pub fn uses(owner: &str, repo: &str, ref_: &str) -> Step<Use> {
         Step::new(function_name(1)).uses(owner, repo, ref_)
     }
 
     /// Returns a bash-script step with the same name as the enclosing function.
     /// (You shouldn't inline this function into the workflow definition, you must
     /// wrap it in a new function.)
-    pub(crate) fn bash(script: &str) -> Step<Run> {
+    pub fn bash(script: &str) -> Step<Run> {
         Step::new(function_name(1)).run(script).shell(BASH_SHELL)
     }
 
     /// Returns a pwsh-script step with the same name as the enclosing function.
     /// (You shouldn't inline this function into the workflow definition, you must
     /// wrap it in a new function.)
-    pub(crate) fn pwsh(script: &str) -> Step<Run> {
+    pub fn pwsh(script: &str) -> Step<Run> {
         Step::new(function_name(1)).run(script).shell(PWSH_SHELL)
     }
 
     /// Runs the command in either powershell or bash, depending on platform.
     /// (You shouldn't inline this function into the workflow definition, you must
     /// wrap it in a new function.)
-    pub(crate) fn run(platform: Platform, script: &str) -> Step<Run> {
+    pub fn run(platform: Platform, script: &str) -> Step<Run> {
         match platform {
             Platform::Windows => Step::new(function_name(1)).run(script).shell(PWSH_SHELL),
             Platform::Linux | Platform::Mac => {
@@ -260,7 +275,7 @@ pub(crate) mod named {
     }
 
     /// Returns a Workflow with the same name as the enclosing module.
-    pub(crate) fn workflow() -> Workflow {
+    pub fn workflow() -> Workflow {
         Workflow::default().name(
             named::function_name(1)
                 .split("::")
@@ -272,7 +287,7 @@ pub(crate) mod named {
 
     /// Returns a Job with the same name as the enclosing function.
     /// (note job names may not contain `::`)
-    pub(crate) fn job(job: Job) -> NamedJob {
+    pub fn job(job: Job) -> NamedJob {
         NamedJob {
             name: function_name(1).split("::").last().unwrap().to_owned(),
             job,
@@ -282,7 +297,7 @@ pub(crate) mod named {
     /// Returns the function name N callers above in the stack
     /// (typically 1).
     /// This only works because xtask always runs debug builds.
-    pub(crate) fn function_name(i: usize) -> String {
+    pub fn function_name(i: usize) -> String {
         let mut name = "<unknown>".to_string();
         let mut count = 0;
         backtrace::trace(|frame| {
@@ -297,6 +312,7 @@ pub(crate) mod named {
             });
             false
         });
+
         name.split("::")
             .skip_while(|s| s != &"workflows")
             .skip(1)

tooling/xtask/src/tasks/workflows/vars.rs

@@ -11,8 +11,8 @@ macro_rules! secret {
 }
 
 macro_rules! var {
-    ($secret_name:ident) => {
-        pub const $secret_name: &str = concat!("${{ vars.", stringify!($secret_name), " }}");
+    ($var_name:ident) => {
+        pub const $var_name: &str = concat!("${{ vars.", stringify!($var_name), " }}");
     };
 }
 
@@ -76,7 +76,7 @@ pub fn bundle_envs(platform: Platform) -> Env {
     }
 }
 
-pub(crate) fn one_workflow_per_non_main_branch() -> Concurrency {
+pub fn one_workflow_per_non_main_branch() -> Concurrency {
     Concurrency::default()
         .group("${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}")
         .cancel_in_progress(true)
@@ -89,7 +89,7 @@ pub(crate) fn allow_concurrent_runs() -> Concurrency {
 }
 
 // Represents a pattern to check for changed files and corresponding output variable
-pub(crate) struct PathCondition {
+pub struct PathCondition {
     pub name: &'static str,
     pub pattern: &'static str,
     pub invert: bool,
@@ -147,6 +147,10 @@ impl StepOutput {
                 .expect("Steps that produce outputs must have an ID"),
         }
     }
+
+    pub fn expr(&self) -> String {
+        format!("steps.{}.outputs.{}", self.step_id, self.name)
+    }
 }
 
 impl serde::Serialize for StepOutput {
@@ -164,7 +168,7 @@ impl std::fmt::Display for StepOutput {
     }
 }
 
-pub(crate) struct Input {
+pub struct Input {
     pub input_type: &'static str,
     pub name: &'static str,
     pub default: Option<String>,