Auto merge of #149921 - Kobzol:src-gpl, r=<try>

Add new source component that includes GPL-licensed source

try-job: dist-x86_64-linux

Author: rust-bors[bot]

src/bootstrap/src/core/build_steps/dist.rs

@@ -1211,6 +1211,9 @@ impl Step for Src {
     }
 }
 
+/// Tarball for people who want to build rustc and other components from the source.
+/// Does not contain GPL code, which is separated into `PlainSourceTarballGpl`
+/// for licensing reasons.
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct PlainSourceTarball;
 
@@ -1233,51 +1236,18 @@ impl Step for PlainSourceTarball {
 
     /// Creates the plain source tarball
     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
-        // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
-        // means neither rustup nor rustup-toolchain-install-master know how to download it.
-        // It also contains symbolic links, unlike other any other dist tarball.
-        // It's used for distros building rustc from source in a pre-vendored environment.
-        let mut tarball = Tarball::new(builder, "rustc", "src");
-        tarball.permit_symlinks(true);
-        let plain_dst_src = tarball.image_dir();
-
-        // This is the set of root paths which will become part of the source package
-        let src_files = [
-            // tidy-alphabetical-start
-            ".gitmodules",
-            "CONTRIBUTING.md",
-            "COPYRIGHT",
-            "Cargo.lock",
-            "Cargo.toml",
-            "LICENSE-APACHE",
-            "LICENSE-MIT",
-            "README.md",
-            "RELEASES.md",
-            "REUSE.toml",
-            "bootstrap.example.toml",
-            "configure",
-            "license-metadata.json",
-            "package.json",
-            "x",
-            "x.ps1",
-            "x.py",
-            "yarn.lock",
-            // tidy-alphabetical-end
-        ];
-        let src_dirs = ["src", "compiler", "library", "tests", "LICENSES"];
-
-        copy_src_dirs(
+        let tarball = prepare_source_tarball(
             builder,
-            &builder.src,
-            &src_dirs,
+            "src",
             &[
                 // We don't currently use the GCC source code for building any official components,
                 // it is very big, and has unclear licensing implications due to being GPL licensed.
                 // We thus exclude it from the source tarball from now.
                 "src/gcc",
             ],
-            plain_dst_src,
         );
+
+        let plain_dst_src = tarball.image_dir();
         // We keep something in src/gcc because it is a registered submodule,
         // and if it misses completely it can cause issues elsewhere
         // (see https://github.com/rust-lang/rust/issues/137332).
@@ -1289,76 +1259,138 @@ impl Step for PlainSourceTarball {
                 "The GCC source code is not included due to unclear licensing implications\n"
             ));
         }
+        tarball.bare()
+    }
+}
 
-        // Copy the files normally
-        for item in &src_files {
-            builder.copy_link(
-                &builder.src.join(item),
-                &plain_dst_src.join(item),
-                FileType::Regular,
-            );
-        }
+/// Tarball with *all* source code for source builds, including GPL-licensed code.
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct PlainSourceTarballGpl;
 
-        // Create the version file
-        builder.create(&plain_dst_src.join("version"), &builder.rust_version());
+impl Step for PlainSourceTarballGpl {
+    /// Produces the location of the tarball generated
+    type Output = GeneratedTarball;
+    const IS_HOST: bool = true;
 
-        // Create the files containing git info, to ensure --version outputs the same.
-        let write_git_info = |info: Option<&Info>, path: &Path| {
-            if let Some(info) = info {
-                t!(std::fs::create_dir_all(path));
-                channel::write_commit_hash_file(path, &info.sha);
-                channel::write_commit_info_file(path, info);
-            }
-        };
-        write_git_info(builder.rust_info().info(), plain_dst_src);
-        write_git_info(builder.cargo_info.info(), &plain_dst_src.join("./src/tools/cargo"));
-
-        if builder.config.dist_vendor {
-            builder.require_and_update_all_submodules();
-
-            // Vendor packages that are required by opt-dist to collect PGO profiles.
-            let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES
-                .iter()
-                .chain(build_helper::RUSTC_PGO_CRATES)
-                .map(|pkg| {
-                    let mut manifest_path =
-                        builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks");
-                    manifest_path.push(pkg);
-                    manifest_path.push("Cargo.toml");
-                    manifest_path
-                });
-
-            // Vendor all Cargo dependencies
-            let vendor = builder.ensure(Vendor {
-                sync_args: pkgs_for_pgo_training.collect(),
-                versioned_dirs: true,
-                root_dir: plain_dst_src.into(),
-                output_dir: VENDOR_DIR.into(),
-            });
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("rustc-src-gpl")
+    }
 
-            let cargo_config_dir = plain_dst_src.join(".cargo");
-            builder.create_dir(&cargo_config_dir);
-            builder.create(&cargo_config_dir.join("config.toml"), &vendor.config);
-        }
+    fn is_default_step(builder: &Builder<'_>) -> bool {
+        builder.config.rust_dist_src
+    }
 
-        // Delete extraneous directories
-        // FIXME: if we're managed by git, we should probably instead ask git if the given path
-        // is managed by it?
-        for entry in walkdir::WalkDir::new(tarball.image_dir())
-            .follow_links(true)
-            .into_iter()
-            .filter_map(|e| e.ok())
-        {
-            if entry.path().is_dir() && entry.path().file_name() == Some(OsStr::new("__pycache__"))
-            {
-                t!(fs::remove_dir_all(entry.path()));
-            }
-        }
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(PlainSourceTarballGpl);
+    }
 
+    /// Creates the plain source tarball
+    fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
+        let tarball = prepare_source_tarball(builder, "src-gpl", &[]);
         tarball.bare()
     }
 }
 
+fn prepare_source_tarball<'a>(
+    builder: &'a Builder<'a>,
+    name: &str,
+    exclude_dirs: &[&str],
+) -> Tarball<'a> {
+    // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
+    // means neither rustup nor rustup-toolchain-install-master know how to download it.
+    // It also contains symbolic links, unlike other any other dist tarball.
+    // It's used for distros building rustc from source in a pre-vendored environment.
+    let mut tarball = Tarball::new(builder, "rustc", name);
+    tarball.permit_symlinks(true);
+    let plain_dst_src = tarball.image_dir();
+
+    // This is the set of root paths which will become part of the source package
+    let src_files = [
+        // tidy-alphabetical-start
+        ".gitmodules",
+        "CONTRIBUTING.md",
+        "COPYRIGHT",
+        "Cargo.lock",
+        "Cargo.toml",
+        "LICENSE-APACHE",
+        "LICENSE-MIT",
+        "README.md",
+        "RELEASES.md",
+        "REUSE.toml",
+        "bootstrap.example.toml",
+        "configure",
+        "license-metadata.json",
+        "package.json",
+        "x",
+        "x.ps1",
+        "x.py",
+        "yarn.lock",
+        // tidy-alphabetical-end
+    ];
+    let src_dirs = ["src", "compiler", "library", "tests", "LICENSES"];
+
+    copy_src_dirs(builder, &builder.src, &src_dirs, exclude_dirs, plain_dst_src);
+
+    // Copy the files normally
+    for item in &src_files {
+        builder.copy_link(&builder.src.join(item), &plain_dst_src.join(item), FileType::Regular);
+    }
+
+    // Create the version file
+    builder.create(&plain_dst_src.join("version"), &builder.rust_version());
+
+    // Create the files containing git info, to ensure --version outputs the same.
+    let write_git_info = |info: Option<&Info>, path: &Path| {
+        if let Some(info) = info {
+            t!(std::fs::create_dir_all(path));
+            channel::write_commit_hash_file(path, &info.sha);
+            channel::write_commit_info_file(path, info);
+        }
+    };
+    write_git_info(builder.rust_info().info(), plain_dst_src);
+    write_git_info(builder.cargo_info.info(), &plain_dst_src.join("./src/tools/cargo"));
+
+    if builder.config.dist_vendor {
+        builder.require_and_update_all_submodules();
+
+        // Vendor packages that are required by opt-dist to collect PGO profiles.
+        let pkgs_for_pgo_training =
+            build_helper::LLVM_PGO_CRATES.iter().chain(build_helper::RUSTC_PGO_CRATES).map(|pkg| {
+                let mut manifest_path =
+                    builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks");
+                manifest_path.push(pkg);
+                manifest_path.push("Cargo.toml");
+                manifest_path
+            });
+
+        // Vendor all Cargo dependencies
+        let vendor = builder.ensure(Vendor {
+            sync_args: pkgs_for_pgo_training.collect(),
+            versioned_dirs: true,
+            root_dir: plain_dst_src.into(),
+            output_dir: VENDOR_DIR.into(),
+        });
+
+        let cargo_config_dir = plain_dst_src.join(".cargo");
+        builder.create_dir(&cargo_config_dir);
+        builder.create(&cargo_config_dir.join("config.toml"), &vendor.config);
+    }
+
+    // Delete extraneous directories
+    // FIXME: if we're managed by git, we should probably instead ask git if the given path
+    // is managed by it?
+    for entry in walkdir::WalkDir::new(tarball.image_dir())
+        .follow_links(true)
+        .into_iter()
+        .filter_map(|e| e.ok())
+    {
+        if entry.path().is_dir() && entry.path().file_name() == Some(OsStr::new("__pycache__")) {
+            t!(fs::remove_dir_all(entry.path()));
+        }
+    }
+    tarball
+}
+
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct Cargo {
     pub build_compiler: Compiler,

src/bootstrap/src/core/builder/cli_paths/snapshots/x_dist.snap

@@ -29,6 +29,9 @@ expression: dist
 [Dist] dist::PlainSourceTarball
     targets: [x86_64-unknown-linux-gnu]
     - Set({dist::rustc-src})
+[Dist] dist::PlainSourceTarballGpl
+    targets: [x86_64-unknown-linux-gnu]
+    - Set({dist::rustc-src-gpl})
 [Dist] dist::ReproducibleArtifacts
     targets: [x86_64-unknown-linux-gnu]
     - Set({dist::reproducible-artifacts})

src/bootstrap/src/core/builder/mod.rs

@@ -986,6 +986,7 @@ impl<'a> Builder<'a> {
                 // and force us to rebuild tools after vendoring dependencies.
                 // To work around this, create the Tarball after building all the tools.
                 dist::PlainSourceTarball,
+                dist::PlainSourceTarballGpl,
                 dist::BuildManifest,
                 dist::ReproducibleArtifacts,
                 dist::Gcc