Support downloading bootstrap from CI in shell scripts

.github/workflows/ci.yml

@@ -316,7 +316,7 @@ jobs:
             os: ubuntu-20.04-xl
           - name: dist-x86_64-apple
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
               RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -328,7 +328,7 @@ jobs:
             os: macos-latest
           - name: dist-apple-various
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
               RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -339,7 +339,7 @@ jobs:
             os: macos-latest
           - name: dist-x86_64-apple-alt
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths"
               RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -372,7 +372,7 @@ jobs:
             os: macos-latest
           - name: dist-aarch64-apple
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2"
+              SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2"
               RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               SELECT_XCODE: /Applications/Xcode_13.4.1.app
@@ -447,33 +447,33 @@ jobs:
           - name: dist-x86_64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler --set rust.lto=thin"
-              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths
+              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl
           - name: dist-i686-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl
           - name: dist-aarch64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
               WINDOWS_SDK_20348_HACK: 1
             os: windows-latest-xl
           - name: dist-i686-mingw
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler"
               NO_DOWNLOAD_CI_LLVM: 1
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl
           - name: dist-x86_64-mingw
             env:
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler"
               NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
@@ -482,7 +482,7 @@ jobs:
           - name: dist-x86_64-msvc-alt
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths
             os: windows-latest-xl
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"

src/bootstrap/Cargo.toml

@@ -14,6 +14,11 @@ name = "bootstrap"
 path = "bin/main.rs"
 test = false
 
+[[bin]]
+name = "bootstrap-shim"
+path = "bin/bootstrap-shim.rs"
+test = false
+
 [[bin]]
 name = "rustc"
 path = "bin/rustc.rs"

src/bootstrap/bin/bootstrap-shim.rs

@@ -0,0 +1,33 @@
+use std::{env, process::Command};
+
+use bootstrap::{t, MinimalConfig};
+
+#[path = "../../../src/tools/x/src/main.rs"]
+mod run_python;
+
+fn main() {
+    let args = env::args().skip(1).collect::<Vec<_>>();
+    let mut opts = getopts::Options::new();
+    opts.optopt("", "config", "TOML configuration file for build", "FILE");
+    let matches = t!(opts.parse(args));
+
+    // If there are no untracked changes to bootstrap, download it from CI.
+    // Otherwise, build it from source. Use python to build to avoid duplicating the code between python and rust.
+    let config = MinimalConfig::parse(t!(matches.opt_get("config")));
+    let bootstrap_bin = if let Some(commit) = last_modified_bootstrap_commit(&config) {
+        config.download_bootstrap(&commit)
+    } else {
+        return run_python::main();
+    };
+
+    let args: Vec<_> = std::env::args().skip(1).collect();
+    Command::new(bootstrap_bin).args(args).status().expect("failed to spawn bootstrap binairy");
+}
+
+fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option<String> {
+    config.last_modified_commit(
+        &["src/bootstrap", "src/tools/build_helper"],
+        "download-bootstrap",
+        true,
+    )
+}

src/bootstrap/builder.rs

@@ -656,7 +656,8 @@ impl<'a> Builder<'a> {
                 check::Rls,
                 check::RustAnalyzer,
                 check::Rustfmt,
-                check::Bootstrap
+                check::Bootstrap,
+                check::BootstrapShim,
             ),
             Kind::Test => describe!(
                 crate::toolstate::ToolStateCheck,
@@ -761,6 +762,7 @@ impl<'a> Builder<'a> {
                 dist::LlvmTools,
                 dist::RustDev,
                 dist::Bootstrap,
+                dist::BootstrapShim,
                 dist::Extended,
                 // It seems that PlainSourceTarball somehow changes how some of the tools
                 // perceive their dependencies (see #93033) which would invalidate fingerprints

src/bootstrap/check.rs

@@ -473,7 +473,9 @@ tool_check_step!(Rls, "src/tools/rls", SourceType::InTree);
 tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree);
 tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree);
 
+// FIXME: currently  these are marked as ToolRustc, but they should be ToolBootstrap instead to avoid having to build the compiler first
 tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false);
+tool_check_step!(BootstrapShim, "src/bootstrap/bin/bootstrap-shim", SourceType::InTree, false);
 
 /// Cargo's output path for the standard library in a given stage, compiled
 /// by a particular compiler for the specified target.