add extra metadata files while building wheels

Author: Shubh Bapna

.github/workflows/test.yaml

@@ -73,6 +73,7 @@ jobs:
           - rust_vendor
           - download_sequence
           - optimize_build
+          - extra_metadata
 
     steps:
       - name: Get source

.mergify.yml

@@ -75,6 +75,9 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, optimize_build)
           - check-success=e2e (3.12, 1.75, optimize_build)
 
+          - check-success=e2e (3.11, 1.75, extra_metadata)
+          - check-success=e2e (3.12, 1.75, extra_metadata)
+
           - "-draft"
 
           # At least 1 reviewer

e2e/test_extra_metadata.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test whether extra metadata was added in the wheels or not
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-file="$SCRIPTDIR/build_settings.yaml" \
+  bootstrap 'stevedore==5.2.0'
+
+find "$OUTDIR/wheels-repo/" -name '*.whl'
+find "$OUTDIR/sdists-repo/" -name '*.tar.gz'
+ls "$OUTDIR"/work-dir/*/build.log || true
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+"
+
+pass=true
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+wheel unpack $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-py3-none-any.whl -d $OUTDIR
+
+EXPECTED_FILES="
+$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/BUILD
+$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/build-backend-requirements.txt
+$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/build-system-requirements.txt
+$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/build-sdist-requirements.txt
+"
+
+pass=true
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+for file in $EXPECTED_FILES; do
+  cat $file
+done

requirements.txt

@@ -14,3 +14,4 @@ stevedore
 tomlkit
 tqdm
 virtualenv
+wheel

src/fromager/wheels.py

@@ -3,13 +3,15 @@
 import os
 import pathlib
 import platform
+import shutil
 import sys
 import tempfile
 import typing
 import zipfile
 from datetime import datetime
 
 import elfdeps
+import tomlkit
 from packaging.requirements import Requirement
 from packaging.utils import parse_wheel_filename
 from packaging.version import Version
@@ -129,6 +131,73 @@ def analyze_wheel_elfdeps(
     return requires, provides
 
 
+def add_extra_metadata_to_wheels(
+    ctx: context.WorkContext,
+    req: Requirement,
+    version: Version,
+    extra_environ: dict[str, str],
+    sdist_root_dir: pathlib.Path,
+    wheel_file: pathlib.Path,
+) -> None:
+    extra_data_plugin = overrides.find_override_method(
+        req.name, "add_extra_metadata_to_wheels"
+    )
+    data_to_add = {}
+    if extra_data_plugin:
+        data_to_add = overrides.invoke(
+            extra_data_plugin,
+            ctx=ctx,
+            req=req,
+            version=version,
+            extra_environ=extra_environ,
+            sdist_root_dir=sdist_root_dir,
+        )
+        if not isinstance(data_to_add, dict):
+            logger.warning(
+                f"{req.name}: unexpected return type from plugin add_extra_metadata_to_wheels. Expected dictionary. Will ignore"
+            )
+            data_to_add = {}
+
+    with tempfile.TemporaryDirectory() as dir_name:
+        cmd = ["wheel", "unpack", str(wheel_file), "-d", dir_name]
+        external_commands.run(
+            cmd,
+            cwd=dir_name,
+            network_isolation=ctx.network_isolation,
+        )
+
+        wheel_file_name_parts = wheel_file.name.split("-")
+        distribution_name = f"{wheel_file_name_parts[0]}-{wheel_file_name_parts[1]}"
+        dist_info_dir = (
+            pathlib.Path(dir_name)
+            / distribution_name
+            / f"{distribution_name}.dist-info"
+        )
+        if not dist_info_dir.exists():
+            logger.debug(
+                f"{req.name}: could not add additional metadata. {dist_info_dir} does not exist"
+            )
+            return
+
+        build_file = dist_info_dir / "BUILD"
+        build_file.write_text(
+            tomlkit.dumps(ctx.settings.get_package_settings(req.name))
+        )
+        if data_to_add:
+            build_file.write_text(tomlkit.dumps(data_to_add))
+
+        req_files = sdist_root_dir.parent.glob("*-requirements.txt")
+        for req_file in req_files:
+            shutil.copy(req_file, dist_info_dir)
+
+        cmd = ["wheel", "pack", str(dist_info_dir.parent), "-d", str(wheel_file.parent)]
+        external_commands.run(
+            cmd,
+            cwd=dir_name,
+            network_isolation=ctx.network_isolation,
+        )
+
+
 def build_wheel(
     ctx: context.WorkContext,
     req: Requirement,
@@ -172,6 +241,14 @@ def build_wheel(
         # TODO: raise error?
         return None
     wheel = wheels[0]
+    add_extra_metadata_to_wheels(
+        ctx=ctx,
+        req=req,
+        version=version,
+        extra_environ=extra_environ,
+        sdist_root_dir=sdist_root_dir,
+        wheel_file=wheel,
+    )
     logger.info(f"{req.name}: built wheel '{wheel}' in {end - start}")
     analyze_wheel_elfdeps(ctx, req, wheel)
     return wheel