diff --git a/Cargo.lock b/Cargo.lock index c9140f0..c0817c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,11 +221,16 @@ dependencies = [ "basalt-bedrock", "bollard", "clap", + "colored", "futures", "lazy_static", "local-ip-address", + "regex", + "tempdir", "tera", "tokio", + "tokio-process-stream", + "tokio-stream", "tokio-tar", ] @@ -493,9 +498,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.45" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" +checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" dependencies = [ "clap_builder", "clap_derive", @@ -503,9 +508,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.44" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" +checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" dependencies = [ "anstream", "anstyle", @@ -552,6 +557,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "colored" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "comemo" version = "0.4.0" @@ -994,6 +1008,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.31" @@ -1842,7 +1862,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "636860251af8963cc40f6b4baadee105f02e21b28131d76eba8e40ce84ab8064" dependencies = [ - "rand", + "rand 0.8.5", "rand_chacha", ] @@ -2299,7 +2319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" dependencies = [ "phf_shared 0.10.0", - "rand", + "rand 0.8.5", ] [[package]] @@ -2309,7 +2329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand", + "rand 0.8.5", ] [[package]] @@ -2547,6 +2567,19 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -2555,7 +2588,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -2565,9 +2598,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", ] +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -2597,6 +2645,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -2648,9 +2705,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -2660,9 +2717,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -2675,6 +2732,15 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "resvg" version = "0.43.0" @@ -3246,6 +3312,16 @@ dependencies = [ "xattr", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.21.0" @@ -3273,7 +3349,7 @@ dependencies = [ "percent-encoding", "pest", "pest_derive", - "rand", + "rand 0.8.5", "regex", "serde", "serde_json", @@ -3471,6 +3547,19 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "tokio-process-stream" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0e450910f9b9b6ec970e872eb4b4264e4294e25b4d1e087a78d26ab86b67790" +dependencies = [ + "futures", + "pin-project-lite", + "tokio", + "tokio-stream", + "tokio-util", +] + [[package]] name = "tokio-stream" version = "0.1.17" diff --git a/Cargo.toml b/Cargo.toml index 582fc81..bd30eec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,3 +33,8 @@ tokio-tar = "0.3.1" futures = "0.3.31" local-ip-address = "0.6.5" ansi_term = "0.12.1" +tokio-process-stream = "0.4.1" +colored = "3.0.0" +tokio-stream = "0.1.17" +tempdir = "0.3.7" +regex = "1.11.3" diff --git a/src/build/containers.rs b/src/build/containers.rs new file mode 100644 index 0000000..da3f44f --- /dev/null +++ b/src/build/containers.rs @@ -0,0 +1,142 @@ +use lazy_static::lazy_static; +use regex::Regex; +use tempdir::TempDir; +use tokio_stream::StreamExt; + +use anyhow::Context; +use bedrock::Config; +use colored::Colorize; +use tokio::{io::AsyncWriteExt, process::Command}; +use tokio_process_stream::ProcessLineStream; +use tokio_tar::Archive; + +use crate::cli::ContainerBackend; + +const APP_VERSION: &str = env!("CARGO_PKG_VERSION"); + +lazy_static! { + static ref ANSI_REGEX: Regex = Regex::new(r"\x1b\[[0-9;]*[A-Za-z]").unwrap(); +} + +pub async fn build_container_image( + tar_bytes: Vec, + tag: String, + container_backend: ContainerBackend, +) -> anyhow::Result<()> { + match container_backend { + ContainerBackend::Docker => { + let docker = bollard::Docker::connect_with_local_defaults() + .context("Failed to connect to docker")?; + let stream = docker.build_image( + bollard::image::BuildImageOptions { + dockerfile: "Dockerfile", + t: &tag, + rm: true, + ..Default::default() + }, + None, + Some(tar_bytes.into()), + ); + + let prefix = "[BUILD]".blue(); + // Process the stream + tokio::pin!(stream); + while let Some(item) = stream.next().await { + let msg = item.context("Failed to perform docker build")?; + if let Some(stream) = msg.stream { + let clean = stream.replace('\r', ""); // strip carriage returns from progress bars + for line in clean.split('\n') { + // remove trailing \n but keep empty lines + let trimmed = line.trim_end_matches('\n'); + if !is_only_formatting_or_whitespace(trimmed) { + println!("{} {}", prefix, trimmed); + } + } + } + } + Ok(()) + } + ContainerBackend::Podman => { + ensure_podman_accessible() + .await + .context("Failed to validate that Podman was accessible")?; + let tmp_dir = TempDir::new("basalt-build").context("Failed to create tempdir")?; + // Unpack tar bytes to temporary directory where we will run `podman build` + let mut ar = Archive::new(tar_bytes.as_slice()); + ar.unpack(&tmp_dir.path()) + .await + .context("Failed to unpack tar to temporary directory")?; + // Build command and convert to stream + let mut build_cmd = Command::new("podman"); + build_cmd + .arg("build") + .arg("-t") + .arg(tag) + .arg(tmp_dir.path()); + let mut stream = ProcessLineStream::try_from(build_cmd) + .context("Failed to create process stream from command")?; + + let prefix = "[BUILD]".blue(); + + // Grab stdout and stderr for better perf + let mut stdout = tokio::io::stdout(); + let mut stderr = tokio::io::stderr(); + while let Some(item) = stream.next().await { + if let Some(out) = item.stdout() { + stdout + .write_all(format!("{} {}\n", prefix, out.clear()).as_bytes()) + .await + .context("Failed to write to STDOUT")?; + } + if let Some(err) = item.stderr() { + stderr + .write_all(format!("{} {}\n", prefix, err).as_bytes()) + .await + .context("Failed to write to STDERR")?; + } + } + Ok(()) + } + } +} + +/// Based on the config, determine which tag to use +pub fn get_server_tag(cfg: &Config) -> String { + let needs_scripting = !cfg.integrations.event_handlers.is_empty(); + let needs_webhooks = !cfg.integrations.webhooks.is_empty(); + let variant = if needs_scripting && needs_webhooks { + "full" + } else if needs_scripting { + "scripting" + } else if needs_webhooks { + "webhooks" + } else { + "minimal" + }; + format!("{APP_VERSION}-{variant}") +} + +pub async fn ensure_podman_accessible() -> anyhow::Result<()> { + Command::new("podman") + .output() + .await + .context("Failed to spawn Podman command")?; + Ok(()) +} + +fn strip_ansi(s: &str) -> String { + // Regex for ANSI escape sequences + ANSI_REGEX.replace_all(s, "").into_owned() +} + +fn is_only_formatting_or_whitespace(s: &str) -> bool { + let stripped = strip_ansi(s); + stripped.chars().all(|c| { + c.is_whitespace() + || c.is_control() + || matches!(c, '\u{200B}'..='\u{200F}' + | '\u{202A}'..='\u{202E}' + | '\u{2060}'..='\u{206F}' + | '\u{FEFF}') + }) +} diff --git a/src/build.rs b/src/build/mod.rs similarity index 51% rename from src/build.rs rename to src/build/mod.rs index c5e0209..9486968 100644 --- a/src/build.rs +++ b/src/build/mod.rs @@ -1,15 +1,19 @@ +use containers::{build_container_image, get_server_tag}; use std::path::{Path, PathBuf}; +use tar_helpers::{append_event_handlers, make_base_init, make_base_install, make_header}; use anyhow::Context; -use bedrock::Config; -use futures::StreamExt; use lazy_static::lazy_static; -use tokio::{io::AsyncReadExt, task::JoinSet}; -use tokio_tar::{Builder, Header}; +use tokio::io::AsyncReadExt; -const BASE_DOCKER_SRC: &str = include_str!("../data/basalt.Dockerfile"); -const INSTALL_SRC: &str = include_str!("../data/install.sh"); -const ENTRY_SRC: &str = include_str!("../data/entrypoint.sh"); +use crate::cli::ContainerBackend; + +mod containers; +mod tar_helpers; + +const BASE_DOCKER_SRC: &str = include_str!("../../data/basalt.Dockerfile"); +const INSTALL_SRC: &str = include_str!("../../data/install.sh"); +const ENTRY_SRC: &str = include_str!("../../data/entrypoint.sh"); const DOCKER_IGNORE: &str = "./Dockerfile\n./.dockerignore"; const APP_VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -31,6 +35,7 @@ pub async fn build_with_output( output: &Option, config_file: &Path, tag: Option, + container_backend: ContainerBackend, ) -> anyhow::Result<()> { let mut file = tokio::fs::File::open(config_file) .await @@ -135,134 +140,13 @@ pub async fn build_with_output( .await .context("Failed to write data")?; } - None => { - let docker = bollard::Docker::connect_with_local_defaults() - .context("Failed to connect to docker")?; - let tag = tag.unwrap_or(format!("bslt-{}", cfg.hash())); - let stream = docker.build_image( - bollard::image::BuildImageOptions { - dockerfile: "Dockerfile", - t: &tag, - rm: true, - ..Default::default() - }, - None, - Some(out_data.into()), - ); - - // Process the stream - tokio::pin!(stream); - while let Some(item) = stream.next().await { - let msg = item.context("Failed to perform docker build")?; - if let Some(stream) = msg.stream { - println!( - "[BUILD] {}", - stream.trim().replace("\n", " ").replace("\t", " ") - ); - } - } - } - }; - Ok(()) -} - -fn make_base_install(cfg: &Config) -> String { - cfg.languages - .iter() - .map(|e| match e { - bedrock::language::Language::BuiltIn { language, version } => { - language.install_command(version).unwrap_or("").to_owned() - } - _ => "".into(), - }) - .filter(|e| !e.is_empty()) - .collect::>() - .join("\n") - .trim() - .to_owned() -} - -fn make_base_init(cfg: &Config) -> String { - cfg.languages - .iter() - .map(|e| match e { - bedrock::language::Language::BuiltIn { language, version } => { - language.init_command(version).unwrap_or("").to_owned() - } - _ => "".into(), - }) - .filter(|e| !e.is_empty()) - .collect::>() - .join("\n") - .trim() - .to_owned() -} - -fn make_header

(path: P, size: u64, mode: u32) -> anyhow::Result

-where - P: AsRef, -{ - let mut header = tokio_tar::Header::new_gnu(); - header - .set_path(&path) - .with_context(|| format!("Failed to set {} tar header", path.as_ref().display()))?; - header.set_size(size); - header.set_mode(mode); - header.set_cksum(); - Ok(header) -} - -/// Based on the config, determine which tag to use -fn get_server_tag(cfg: &Config) -> String { - let needs_scripting = !cfg.integrations.event_handlers.is_empty(); - let needs_webhooks = !cfg.integrations.webhooks.is_empty(); - let variant = if needs_scripting && needs_webhooks { - "full" - } else if needs_scripting { - "scripting" - } else if needs_webhooks { - "webhooks" - } else { - "minimal" - }; - format!("{APP_VERSION}-{variant}") -} - -async fn append_event_handlers(tb: &mut Builder>, cfg: Config) -> anyhow::Result<()> { - let mut set = JoinSet::new(); - - for handler_path in cfg.integrations.event_handlers { - set.spawn(async move { - let contents = tokio::fs::read_to_string(&handler_path) - .await - .with_context(|| { - format!( - "Failed to read script contents from {}", - handler_path.display() - ) - })?; - - Ok::<_, anyhow::Error>((handler_path, contents)) - }); - } - - // Collect results (unordered) - let results = set - .join_all() + None => build_container_image( + out_data, + tag.unwrap_or_else(|| format!("bslt-{}", cfg.hash())), + container_backend, + ) .await - .into_iter() - .collect::, _>>() - .context("Failed to read scripts")?; - - // Append sequentially (tarball writes must be ordered) - for (handler_path, contents) in results { - let script_header = make_header(&handler_path, contents.len() as u64, 0o644) - .context("Failed to create script header")?; - - tb.append(&script_header, contents.as_bytes()) - .await - .context("Failed to append script to tarball")?; - } - + .context("Failed to build container image")?, + }; Ok(()) } diff --git a/src/build/tar_helpers.rs b/src/build/tar_helpers.rs new file mode 100644 index 0000000..ff31c04 --- /dev/null +++ b/src/build/tar_helpers.rs @@ -0,0 +1,91 @@ +use std::path::{Path, PathBuf}; + +use anyhow::Context; +use bedrock::Config; +use tokio::task::JoinSet; +use tokio_tar::{Builder, Header}; + +pub fn make_base_install(cfg: &Config) -> String { + cfg.languages + .iter() + .map(|e| match e { + bedrock::language::Language::BuiltIn { language, version } => { + language.install_command(version).unwrap_or("").to_owned() + } + _ => "".into(), + }) + .filter(|e| !e.is_empty()) + .collect::>() + .join("\n") + .trim() + .to_owned() +} + +pub fn make_base_init(cfg: &Config) -> String { + cfg.languages + .iter() + .map(|e| match e { + bedrock::language::Language::BuiltIn { language, version } => { + language.init_command(version).unwrap_or("").to_owned() + } + _ => "".into(), + }) + .filter(|e| !e.is_empty()) + .collect::>() + .join("\n") + .trim() + .to_owned() +} + +pub fn make_header

(path: P, size: u64, mode: u32) -> anyhow::Result

+where + P: AsRef, +{ + let mut header = tokio_tar::Header::new_gnu(); + header + .set_path(&path) + .with_context(|| format!("Failed to set {} tar header", path.as_ref().display()))?; + header.set_size(size); + header.set_mode(mode); + header.set_cksum(); + Ok(header) +} + +pub async fn append_event_handlers(tb: &mut Builder>, cfg: Config) -> anyhow::Result<()> { + let mut set = JoinSet::new(); + + for handler_path in cfg.integrations.event_handlers { + set.spawn(async move { + let contents = tokio::fs::read_to_string(&handler_path) + .await + .with_context(|| { + format!( + "Failed to read script contents from {}", + handler_path.display() + ) + })?; + + Ok::<_, anyhow::Error>((handler_path, contents)) + }); + } + + // Collect results (unordered) + let results = set + .join_all() + .await + .into_iter() + .collect::, _>>() + .context("Failed to read scripts")?; + + // Append sequentially (tarball writes must be ordered) + for (handler_path, contents) in results { + let script_header = make_header(&handler_path, contents.len() as u64, 0o644) + .context("Failed to create script header")?; + + tb.append(&script_header, contents.as_bytes()) + .await + .context("Failed to append script to tarball")?; + } + + Ok(()) +} diff --git a/src/cli.rs b/src/cli.rs index 9d39ec5..3a50cf4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,11 +1,17 @@ use std::{net::Ipv4Addr, path::PathBuf}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; fn default_config() -> &'static std::ffi::OsStr { std::ffi::OsStr::new("basalt.toml") } +#[derive(Clone, Debug, PartialEq, Eq, Hash, ValueEnum)] +pub enum ContainerBackend { + Docker, + Podman, +} + #[derive(Clone, Debug, Subcommand, PartialEq, Eq, Hash)] pub enum SubCmd { /// Verify that the configuration in a configuration file is correct without attempting to @@ -32,6 +38,9 @@ pub enum SubCmd { output: Option, /// The configuration file to build config_file: PathBuf, + /// The backend to use to build container + #[arg(long, value_enum, default_value_t = ContainerBackend::Docker)] + container_backend: ContainerBackend, }, /// Build the docker file based on a given configuration file and then run it using docker. Run { diff --git a/src/main.rs b/src/main.rs index a773b12..403cc3b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,7 +52,8 @@ async fn main() -> anyhow::Result<()> { tag, output, config_file, - } => build_with_output(&output, &config_file, tag).await?, + container_backend, + } => build_with_output(&output, &config_file, tag, container_backend).await?, cli::SubCmd::Run { .. } => { todo!(); }