Rollup merge of #133208 - ferrocene:split-copyright-html, r=Kobzol

generate-copyright: Now generates a library file too.

We only run reuse once, so the output has to be filtered to find only the files that are relevant to the library tree.

Outputs COPYRIGHT.html and COPYRIGHT-library.html.

The license-metadata.json file is also now in the tree. We need a CI tool to check that it's correct.

r? kobzol

Remaining steps:

* [ ] Teach CI to double-check the license-metadata.json file is correct
* [ ] Add the COPYRIGHT.html and COPYRIGHT-license.html to the releases

Author: compiler-errors

src/bootstrap/src/core/build_steps/run.rs

@@ -211,12 +211,13 @@ impl Step for GenerateCopyright {
     fn run(self, builder: &Builder<'_>) -> Self::Output {
         let license_metadata = builder.ensure(CollectLicenseMetadata);
 
-        // Temporary location, it will be moved to the proper one once it's accurate.
         let dest = builder.out.join("COPYRIGHT.html");
+        let dest_libstd = builder.out.join("COPYRIGHT-library.html");
 
         let mut cmd = builder.tool_cmd(Tool::GenerateCopyright);
         cmd.env("LICENSE_METADATA", &license_metadata);
         cmd.env("DEST", &dest);
+        cmd.env("DEST_LIBSTD", &dest_libstd);
         cmd.env("OUT_DIR", &builder.out);
         cmd.env("CARGO", &builder.initial_cargo);
         cmd.run(builder);

src/tools/generate-copyright/src/cargo_metadata.rs

@@ -45,21 +45,20 @@ pub struct PackageMetadata {
 
 /// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data.
 ///
-/// This will involve running `cargo vendor` into `${BUILD}/vendor` so we can
+/// This will involve running `cargo vendor` into `vendor_path` so we can
 /// grab the license files.
 ///
 /// Any dependency with a path beginning with `root_path` is ignored, as we
 /// assume `reuse` has covered it already.
 pub fn get_metadata_and_notices(
     cargo: &Path,
-    dest: &Path,
+    vendor_path: &Path,
     root_path: &Path,
     manifest_paths: &[&Path],
 ) -> Result<BTreeMap<Package, PackageMetadata>, Error> {
     let mut output = get_metadata(cargo, root_path, manifest_paths)?;
 
     // Now do a cargo-vendor and grab everything
-    let vendor_path = dest.join("vendor");
     println!("Vendoring deps into {}...", vendor_path.display());
     run_cargo_vendor(cargo, &vendor_path, manifest_paths)?;
 

src/tools/generate-copyright/src/main.rs

@@ -6,63 +6,96 @@ use rinja::Template;
 
 mod cargo_metadata;
 
-#[derive(Template)]
-#[template(path = "COPYRIGHT.html")]
-struct CopyrightTemplate {
-    in_tree: Node,
-    dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
-}
-
 /// The entry point to the binary.
 ///
 /// You should probably let `bootstrap` execute this program instead of running it directly.
 ///
 /// Run `x.py run generate-copyright`
 fn main() -> Result<(), Error> {
     let dest_file = env_path("DEST")?;
+    let libstd_dest_file = env_path("DEST_LIBSTD")?;
     let out_dir = env_path("OUT_DIR")?;
     let cargo = env_path("CARGO")?;
     let license_metadata = env_path("LICENSE_METADATA")?;
 
-    let collected_tree_metadata: Metadata =
-        serde_json::from_slice(&std::fs::read(&license_metadata)?)?;
-
     let root_path = std::path::absolute(".")?;
-    let workspace_paths = [
-        Path::new("./Cargo.toml"),
-        Path::new("./src/tools/cargo/Cargo.toml"),
-        Path::new("./library/Cargo.toml"),
-    ];
-    let mut collected_cargo_metadata =
-        cargo_metadata::get_metadata_and_notices(&cargo, &out_dir, &root_path, &workspace_paths)?;
 
-    let stdlib_set =
-        cargo_metadata::get_metadata(&cargo, &root_path, &[Path::new("./library/std/Cargo.toml")])?;
+    // Scan Cargo dependencies
+    let mut collected_cargo_metadata =
+        cargo_metadata::get_metadata_and_notices(&cargo, &out_dir.join("vendor"), &root_path, &[
+            Path::new("./Cargo.toml"),
+            Path::new("./src/tools/cargo/Cargo.toml"),
+            Path::new("./library/Cargo.toml"),
+        ])?;
+
+    let library_collected_cargo_metadata = cargo_metadata::get_metadata_and_notices(
+        &cargo,
+        &out_dir.join("library-vendor"),
+        &root_path,
+        &[Path::new("./library/Cargo.toml")],
+    )?;
 
     for (key, value) in collected_cargo_metadata.iter_mut() {
-        value.is_in_libstd = Some(stdlib_set.contains_key(key));
+        value.is_in_libstd = Some(library_collected_cargo_metadata.contains_key(key));
     }
 
+    // Load JSON output by reuse
+    let collected_tree_metadata: Metadata =
+        serde_json::from_slice(&std::fs::read(&license_metadata)?)?;
+
+    // Find libstd sub-set
+    let library_collected_tree_metadata = Metadata {
+        files: collected_tree_metadata
+            .files
+            .trim_clone(&Path::new("./library"), &Path::new("."))
+            .unwrap(),
+    };
+
+    // Output main file
     let template = CopyrightTemplate {
         in_tree: collected_tree_metadata.files,
         dependencies: collected_cargo_metadata,
     };
-
     let output = template.render()?;
-
     std::fs::write(&dest_file, output)?;
 
+    // Output libstd subset file
+    let template = LibraryCopyrightTemplate {
+        in_tree: library_collected_tree_metadata.files,
+        dependencies: library_collected_cargo_metadata,
+    };
+    let output = template.render()?;
+    std::fs::write(&libstd_dest_file, output)?;
+
     Ok(())
 }
 
+/// The HTML template for the toolchain copyright file
+#[derive(Template)]
+#[template(path = "COPYRIGHT.html")]
+struct CopyrightTemplate {
+    in_tree: Node,
+    dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
+}
+
+/// The HTML template for the library copyright file
+#[derive(Template)]
+#[template(path = "COPYRIGHT-library.html")]
+struct LibraryCopyrightTemplate {
+    in_tree: Node,
+    dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
+}
+
 /// Describes a tree of metadata for our filesystem tree
-#[derive(serde::Deserialize)]
+///
+/// Must match the JSON emitted by the `CollectLicenseMetadata` bootstrap tool.
+#[derive(serde::Deserialize, Clone, Debug, PartialEq, Eq)]
 struct Metadata {
     files: Node,
 }
 
 /// Describes one node in our metadata tree
-#[derive(serde::Deserialize, rinja::Template)]
+#[derive(serde::Deserialize, rinja::Template, Clone, Debug, PartialEq, Eq)]
 #[serde(rename_all = "kebab-case", tag = "type")]
 #[template(path = "Node.html")]
 pub(crate) enum Node {
@@ -72,8 +105,74 @@ pub(crate) enum Node {
     Group { files: Vec<String>, directories: Vec<String>, license: License },
 }
 
+impl Node {
+    /// Clone, this node, but only if the path to the item is within the match path
+    fn trim_clone(&self, match_path: &Path, parent_path: &Path) -> Option<Node> {
+        match self {
+            Node::Root { children } => {
+                let mut filtered_children = Vec::new();
+                for node in children {
+                    if let Some(child_node) = node.trim_clone(match_path, parent_path) {
+                        filtered_children.push(child_node);
+                    }
+                }
+                if filtered_children.is_empty() {
+                    None
+                } else {
+                    Some(Node::Root { children: filtered_children })
+                }
+            }
+            Node::Directory { name, children, license } => {
+                let child_name = parent_path.join(name);
+                if !(child_name.starts_with(match_path) || match_path.starts_with(&child_name)) {
+                    return None;
+                }
+                let mut filtered_children = Vec::new();
+                for node in children {
+                    if let Some(child_node) = node.trim_clone(match_path, &child_name) {
+                        filtered_children.push(child_node);
+                    }
+                }
+                Some(Node::Directory {
+                    name: name.clone(),
+                    children: filtered_children,
+                    license: license.clone(),
+                })
+            }
+            Node::File { name, license } => {
+                let child_name = parent_path.join(name);
+                if !(child_name.starts_with(match_path) || match_path.starts_with(&child_name)) {
+                    return None;
+                }
+                Some(Node::File { name: name.clone(), license: license.clone() })
+            }
+            Node::Group { files, directories, license } => {
+                let mut filtered_child_files = Vec::new();
+                for child in files {
+                    let child_name = parent_path.join(child);
+                    if child_name.starts_with(match_path) || match_path.starts_with(&child_name) {
+                        filtered_child_files.push(child.clone());
+                    }
+                }
+                let mut filtered_child_dirs = Vec::new();
+                for child in directories {
+                    let child_name = parent_path.join(child);
+                    if child_name.starts_with(match_path) || match_path.starts_with(&child_name) {
+                        filtered_child_dirs.push(child.clone());
+                    }
+                }
+                Some(Node::Group {
+                    files: filtered_child_files,
+                    directories: filtered_child_dirs,
+                    license: license.clone(),
+                })
+            }
+        }
+    }
+}
+
 /// A License has an SPDX license name and a list of copyright holders.
-#[derive(serde::Deserialize)]
+#[derive(serde::Deserialize, Clone, Debug, PartialEq, Eq)]
 struct License {
     spdx: String,
     copyright: Vec<String>,

src/tools/generate-copyright/templates/COPYRIGHT-library.html

@@ -0,0 +1,53 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Copyright notices for The Rust Standard Library</title>
+</head>
+<body>
+
+<h1>Copyright notices for The Rust Standard Library</h1>
+
+<p>This file describes the copyright and licensing information for the Rust
+Standard Library source code within The Rust Project git tree, and the
+third-party dependencies used when building the Rust Standard Library.</p>
+
+<h2>Table of Contents</h2>
+<ul>
+    <li><a href="#in-tree-files">In-tree files</a></li>
+    <li><a href="#out-of-tree-dependencies">Out-of-tree dependencies</a></li>
+</ul>
+
+<h2 id="in-tree-files">In-tree files</h2>
+
+<p>The following licenses cover the in-tree source files that were used in this
+release:</p>
+
+{{ in_tree|safe }}
+
+<h2 id="out-of-tree-dependencies">Out-of-tree dependencies</h2>
+
+<p>The following licenses cover the out-of-tree crates that were used in the
+Rust Standard Library in this release:</p>
+
+{% for (key, value) in dependencies %}
+    <h3>📦 {{key.name}}-{{key.version}}</h3>
+    <p><b>URL:</b> <a href="https://crates.io/crates/{{ key.name }}/{{ key.version }}">https://crates.io/crates/{{ key.name }}/{{ key.version }}</a></p>
+    <p><b>Authors:</b> {{ value.authors|join(", ") }}</p>
+    <p><b>License:</b> {{ value.license }}</p>
+    {% let len = value.notices.len() %}
+    {% if len > 0 %}
+        <p><b>Notices:</b>
+        {% for (notice_name, notice_text) in value.notices %}
+            <details>
+                <summary><code>{{ notice_name }}</code></summary>
+                <pre>
+{{ notice_text }}
+                </pre>
+            </details>
+        {% endfor %}
+        </p>
+    {% endif %}
+{% endfor %}
+</body>
+</html>