Merge pull request #1895 from EliahKagan/run-ci/s390x

Set up `cross` for limited but substantial local testing of s390x and Android

Author: EliahKagan

etc/docker/Dockerfile.test-cross

@@ -0,0 +1,7 @@
+ARG TARGET
+FROM ghcr.io/cross-rs/${TARGET}:latest
+
+ARG TARGET
+COPY customize.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/customize.sh && \
+    /usr/local/bin/customize.sh "$TARGET"

etc/docker/test-cross-context/customize.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+set -euxC
+
+target="$1"
+test -n "$target"
+
+# Arrange for the indirect `tzdata` dependency to be installed and configured
+# without prompting for the time zone. (Passing `-y` is not enough.)
+export DEBIAN_FRONTEND=noninteractive TZ=UTC
+
+# Install tools for setting up APT repositores. Install `apt-utils` before the
+# others, so the installation of `gnupg` can use it for debconf.
+apt-get update
+apt-get install --no-install-recommends -y apt-utils
+apt-get install --no-install-recommends -y apt-transport-https dpkg-dev gnupg
+type dpkg-architecture  # Make sure we really have this.
+
+# Decide what architecture to use for `git`, shared libraries `git` links to,
+# and shared libraries gitoxide links to when building `max`. Instead of this
+# custom logic, we could use `$CROSS_DEB_ARCH`, which `cross` tries to provide
+# (https://github.com/cross-rs/cross/blob/v0.2.5/src/lib.rs#L268), and which is
+# available for roughly the same architectures where this logic gets a nonempty
+# value. But using `$CROSS_DEB_ARCH` may make it harder to build and test the
+# image manually. In particular, if it is not passed, we would conclude that we
+# should install the versions of those packages with the host's architecture.
+apt_suffix=
+if target_arch="$(dpkg-architecture --host-type "$target" --query DEB_HOST_ARCH)"
+then
+    dpkg --add-architecture "$target_arch"
+    apt_suffix=":$target_arch"
+    printf 'INFO: Using target architecture for `git` and libs in container.\n'
+    printf 'INFO: This architecture is %s.\n' "$target_arch"
+else
+    apt_suffix=''
+    printf 'WARNING: Using HOST architecture for `git` and libs in container.\n'
+fi
+
+# Get release codename. Like `lsb_release -sc`. (`lsb_release` may be absent.)
+release="$(sed -n 's/^VERSION_CODENAME=//p' /etc/os-release)"
+
+# Add the git-core PPA manually. (Faster than installing `add-apt-repository`.)
+echo "deb https://ppa.launchpadcontent.net/git-core/ppa/ubuntu $release main" \
+    >/etc/apt/sources.list.d/git-core-ubuntu-ppa.list
+apt-key adv --keyserver keyserver.ubuntu.com \
+    --recv-keys F911AB184317630C59970973E363C90F8F1B6217
+apt-get update
+
+# Remove the old `git` and associated packages.
+apt-get purge --autoremove -y git
+
+# Git dependencies. These are for the desired architecture, except `git-man` is
+# the same package for all architectures, and we can't always install `perl` or
+# `liberror-perl` for the desired architecture (at least in s390x).
+# TODO(maint): Resolve these dynamically to support future `cross` base images.
+git_deps=(
+    git-man
+    "libc6$apt_suffix"
+    "libcurl3-gnutls$apt_suffix"
+    "libexpat1$apt_suffix"
+    liberror-perl
+    "libpcre2-8-0$apt_suffix"
+    "zlib1g$apt_suffix"
+    perl
+)
+
+# Other dependencies for running the gitoxide test suite and fixture scripts,
+# and for building and testing gitoxide for feature sets beyond `max-pure`.
+gix_test_deps=(
+    ca-certificates
+    cmake
+    "curl$apt_suffix"
+    jq
+    "libc-dev$apt_suffix"
+    "libssl-dev$apt_suffix"
+    patch
+    pkgconf
+)
+
+if test -n "$apt_suffix"; then
+    # Install everything we need except `git` (and what we already have). We
+    # can't necessarily install `git` this way, because it insists on `perl`
+    # and `liberror-perl` dependencies of the same architecture as it. These
+    # may not be possible to install in a mixed environment, where most
+    # packages are a different architecture, and where `perl` is a dependency
+    # of other important packages. So we will install everything else first
+    # (then manually add `git`).
+    apt-get install --no-install-recommends -y \
+        "${git_deps[@]}" "${gix_test_deps[@]}" file
+
+    # Add `git` by manually downloading it and installing it with `dpkg`,
+    # forcing installation to proceed even if its `perl` and `liberror-perl`
+    # dependencies, as declared by `git`, are absent. (We have already
+    # installed them, but in a possibly different architecture. `git` can still
+    # use them, because its use is to run scripts, rather than to link to a
+    # shared library they provide.) It is preferred to let `apt-get download`
+    # drop privileges to the `_apt` user during download, so we download it
+    # inside `/tmp`. But we create a subdirectory so it is safe to make
+    # assumptions about what files globs can expand to, even if `/tmp` is
+    # mounted to an outside share temp dir on a multi-user system.
+    mkdir /tmp/dl  # Don't use `-p`; if it exists already, we cannot trust it.
+    chown _apt /tmp/dl  # Use owner, as the container may have no `_apt` group.
+    (cd /tmp/dl && apt-get download "git$apt_suffix")
+    dpkg --ignore-depends="perl$apt_suffix,liberror-perl$apt_suffix" \
+        -i /tmp/dl/git[-_]*.deb
+    rm -r /tmp/dl
+else
+    # Install everything we need, including `git`.
+    apt-get install --no-install-recommends -y git "${gix_test_deps[@]}" file
+fi
+
+# Show information about the newly installed `git` (and ensure it can run).
+git version --build-options
+git="$(command -v git)"
+file -- "$git"
+
+# Clean up files related to package management that we won't need anymore.
+apt-get clean
+rm -rf /var/lib/apt/lists/*
+
+# If this image has a runner script `cross` uses for Android, patch the script
+# to add the ability to suppress its customization of `LD_PRELOAD`. The runner
+# script sets `LD_PRELOAD` to the path of `libc++_shared.so` in the Android NDK
+# (https://github.com/cross-rs/cross/blob/v0.2.5/docker/android-runner#L34).
+# But this causes a problem for us. When a host-architecture program is run,
+# `ld.so` shows a message about the "wrong ELF class". Such programs can still
+# run, but when we rely on their specific output to stderr, fixtures and tests
+# fail. The change we make here lets us set `NO_PRELOAD_CXX=1` to avoid that.
+runner=/android-runner
+patch='s/^[[:blank:]]*export LD_PRELOAD=/test "${NO_PRELOAD_CXX:-0}" != 0 || &/'
+if test -f "$runner"; then sed -i.orig "$patch" -- "$runner"; fi
+
+# Ensure a nonempty Git `system` scope (for the `installation_config` tests).
+git config --system gitoxide.imaginary.arbitraryVariable arbitraryValue

etc/docker/test-cross.toml

@@ -0,0 +1,26 @@
+# `cross` configuration for running tests. Treated like `Cross.toml` if enabled
+# with `CROSS_CONFIG=etc/docker/test-cross.toml`. This avoids affecting other
+# `cross` usage, e.g. in `release.yml`. See `cross-test` recipes in `justfile`.
+
+[build.env]
+passthrough = [
+    "CI",
+    "GITHUB_ACTIONS",
+    "GIX_CREDENTIALS_HELPER_STDERR",
+    "GIX_EXTERNAL_COMMAND_STDERR",
+    "GIX_OBJECT_CACHE_MEMORY",
+    "GIX_PACK_CACHE_MEMORY",
+    "GIX_TEST_CREATE_ARCHIVES_EVEN_ON_CI",
+    "GIX_TEST_EXPECT_REDUCED_TRUST",
+    "GIX_TEST_IGNORE_ARCHIVES",
+    "GIX_VERSION",
+    "NO_PRELOAD_CXX",
+    "RUST_BACKTRACE",
+    "RUST_LIB_BACKTRACE",
+]
+
+[target.armv7-linux-androideabi]
+image = "cross-rs-gitoxide:armv7-linux-androideabi"
+
+[target.s390x-unknown-linux-gnu]
+image = "cross-rs-gitoxide:s390x-unknown-linux-gnu"

gix-config-value/tests/value/path.rs

@@ -89,16 +89,16 @@ mod interpolate {
         Ok(())
     }
 
-    #[cfg(windows)]
+    #[cfg(any(target_os = "windows", target_os = "android"))]
     #[test]
-    fn tilde_with_given_user_is_unsupported_on_windows() {
+    fn tilde_with_given_user_is_unsupported_on_windows_and_android() {
         assert!(matches!(
             interpolate_without_context("~baz/foo/bar"),
             Err(gix_config_value::path::interpolate::Error::UserInterpolationUnsupported)
         ));
     }
 
-    #[cfg(not(windows))]
+    #[cfg(not(any(target_os = "windows", target_os = "android")))]
     #[test]
     fn tilde_with_given_user() -> crate::Result {
         let home = std::env::current_dir()?;

justfile

@@ -224,6 +224,25 @@ journey-tests-async: dbg
     cargo build -p gix-testtools
     dbg="$({{ j }} dbg)" && tests/journey.sh "$dbg/ein" "$dbg/gix" "$dbg/jtt" async
 
+# Build a customized `cross` container image for testing
+cross-image target:
+    docker build --build-arg "TARGET={{ target }}" \
+        -t "cross-rs-gitoxide:{{ target }}" \
+        -f etc/docker/Dockerfile.test-cross etc/docker/test-cross-context
+
+# Test another platform with `cross`
+cross-test target options test-options: (cross-image target)
+    CROSS_CONFIG=etc/docker/test-cross.toml NO_PRELOAD_CXX=1 \
+        cross test --workspace --no-fail-fast --target {{ target }} \
+        {{ options }} -- --skip realpath::fuzzed_timeout {{ test-options }}
+
+# Test s390x with `cross`
+cross-test-s390x: (cross-test 's390x-unknown-linux-gnu' '' '')
+
+# Test Android with `cross` (max-pure)
+cross-test-android: (cross-test 'armv7-linux-androideabi'
+                     '--no-default-features --features max-pure' '')
+
 # Run `cargo diet` on all crates to see that they are still in bounds
 check-size:
     etc/check-package-size.sh

tests/it/src/args.rs

@@ -73,6 +73,22 @@ pub enum Subcommands {
     /// current repository. Its main use is checking that fixture scripts are have correct modes.
     #[clap(visible_alias = "cm")]
     CheckMode {},
+    /// Print environment variables as `NAME=value` lines.
+    ///
+    /// It is useful to be able to observe environment variables that are set when running code
+    /// with tools such as `cargo` or `cross`. Commands like `cargo run -p internal-tools -- env`
+    /// include environment changes from `cargo` itself. With `cross`, changes are more extensive,
+    /// due to effects of `build.env.passthrough`, container customization, and preexisting special
+    /// cases in wrapper scripts shipped in default `cross` containers (such as to `LD_PRELOAD`).
+    ///
+    /// Since one use for checking environment variables is to investigate the effects of
+    /// environments that contain variable names or values that are not valid Unicode, this avoids
+    /// requiring that environment variables all be Unicode. Any name or value that is not Unicode
+    /// is shown in its Rust debug representation. This is always quoted. To decrease ambiguity,
+    /// any name or value containing a literal double quote or newline is also shown in its debug
+    /// representation. Names and values without such content are shown literally and not quoted.
+    #[clap(visible_alias = "e")]
+    Env {},
 }
 
 #[derive(Clone)]

tests/it/src/commands/env.rs

@@ -0,0 +1,14 @@
+pub(super) mod function {
+    pub fn env() -> anyhow::Result<()> {
+        for (name, value) in std::env::vars_os() {
+            println!("{}={}", repr(&name), repr(&value));
+        }
+        Ok(())
+    }
+
+    fn repr(text: &std::ffi::OsStr) -> String {
+        text.to_str()
+            .filter(|s| !s.chars().any(|c| c == '"' || c == '\n'))
+            .map_or_else(|| format!("{text:?}"), ToOwned::to_owned)
+    }
+}

tests/it/src/commands/mod.rs

@@ -6,3 +6,6 @@ pub use git_to_sh::function::git_to_sh;
 
 pub mod check_mode;
 pub use check_mode::function::check_mode;
+
+pub mod env;
+pub use env::function::env;

tests/it/src/main.rs

@@ -32,6 +32,7 @@ fn main() -> anyhow::Result<()> {
             patterns,
         } => commands::copy_royal(dry_run, &worktree_root, destination_dir, patterns),
         Subcommands::CheckMode {} => commands::check_mode(),
+        Subcommands::Env {} => commands::env(),
     }
 }
 

tests/tools/src/lib.rs

@@ -992,7 +992,7 @@ pub fn umask() -> u32 {
         .output()
         .expect("can execute `sh -c umask`");
     assert!(output.status.success(), "`sh -c umask` failed");
-    assert!(output.stderr.is_empty(), "`sh -c umask` unexpected message");
+    assert_eq!(output.stderr.as_bstr(), "", "`sh -c umask` unexpected message");
     let text = output.stdout.to_str().expect("valid Unicode").trim();
     u32::from_str_radix(text, 8).expect("parses as octal number")
 }

tests/tools/tests/umask.rs

@@ -1,6 +1,9 @@
 #[test]
 #[cfg(unix)]
-#[cfg_attr(not(target_os = "linux"), ignore = "The test itself uses /proc")]
+#[cfg_attr(
+    not(any(target_os = "linux", target_os = "android")),
+    ignore = "The test itself uses /proc"
+)]
 fn umask() {
     use std::fs::File;
     use std::io::{BufRead, BufReader};