From ccb5007a2ef77289aa75e8a3274ba2c21194233a Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 24 Jan 2024 14:53:37 -0800 Subject: [PATCH] new: Add download/unpack extensions. (#1) * Add impl. * Update macros. * Add common. * Add dl crate. * Polish. * Use crates. * Add download tests. * Rename crates. * Add unpack tests. * Add github. * Disable tests. * Add features. --- .github/FUNDING.yml | 1 + .github/workflows/ci.yml | 53 ++++++ .github/workflows/release.yml | 31 ++++ Cargo.toml | 11 +- crates/{dlu => common}/Cargo.toml | 3 +- crates/common/src/download.rs | 30 ++++ crates/common/src/lib.rs | 1 + crates/dlu/src/lib.rs | 2 - crates/download/CHANGELOG.md | 5 + crates/download/Cargo.toml | 22 +++ crates/download/src/download_ext.rs | 68 ++++++++ crates/download/src/lib.rs | 5 + crates/download/tests/download_test.rs | 109 ++++++++++++ crates/unpack/CHANGELOG.md | 5 + crates/unpack/Cargo.toml | 26 +++ crates/unpack/src/lib.rs | 5 + crates/unpack/src/unpack_ext.rs | 119 +++++++++++++ .../unpack/tests/__fixtures__/tar/archive.tar | Bin 0 -> 2560 bytes .../tests/__fixtures__/tar/archive.tar.gz | Bin 0 -> 170 bytes .../unpack/tests/__fixtures__/zip/archive.zip | Bin 0 -> 373 bytes crates/unpack/tests/unpack_test.rs | 158 ++++++++++++++++++ 21 files changed, 650 insertions(+), 4 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml rename crates/{dlu => common}/Cargo.toml (75%) create mode 100644 crates/common/src/download.rs create mode 100644 crates/common/src/lib.rs delete mode 100644 crates/dlu/src/lib.rs create mode 100644 crates/download/CHANGELOG.md create mode 100644 crates/download/Cargo.toml create mode 100644 crates/download/src/download_ext.rs create mode 100644 crates/download/src/lib.rs create mode 100644 crates/download/tests/download_test.rs create mode 100644 crates/unpack/CHANGELOG.md create mode 100644 crates/unpack/Cargo.toml create mode 100644 crates/unpack/src/lib.rs create mode 100644 crates/unpack/src/unpack_ext.rs create mode 100644 crates/unpack/tests/__fixtures__/tar/archive.tar create mode 100644 crates/unpack/tests/__fixtures__/tar/archive.tar.gz create mode 100644 crates/unpack/tests/__fixtures__/zip/archive.zip create mode 100644 crates/unpack/tests/unpack_test.rs diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..9139d92 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: moonrepo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..db64511 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + +jobs: + format: + name: Format + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: moonrepo/setup-rust@v1 + with: + cache: false + components: rustfmt + - run: cargo fmt --all --check + lint: + name: Lint + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: moonrepo/setup-rust@v1 + with: + cache: false + components: clippy + - run: cargo clippy --workspace --all-targets + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: moonrepo/setup-rust@v1 + with: + bins: cargo-nextest + cache: false + - uses: moonrepo/setup-toolchain@v0 + - uses: moonrepo/build-wasm-plugin@v0 + - run: cargo nextest run --no-default-features diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e2889e0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +name: Release + +permissions: + contents: write + +on: + push: + tags: + - "[a-z0-9_]+-v[0-9]+*" + pull_request: + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: moonrepo/setup-rust@v1 + with: + cache: false + - id: build + uses: moonrepo/build-wasm-plugin@v0 + - if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} + uses: ncipollo/release-action@v1 + with: + artifacts: builds/* + artifactErrorsFailBuild: true + body: ${{ steps.build.outputs.changelog-entry }} + makeLatest: true + prerelease: ${{ contains(github.ref_name, '-alpha') || contains(github.ref_name, '-beta') || contains(github.ref_name, '-rc') }} + skipIfReleaseExists: true diff --git a/Cargo.toml b/Cargo.toml index 9e5f6dc..ea63b8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,5 +3,14 @@ resolver = "2" members = ["crates/*"] [workspace.dependencies] -moon_pdk = { path = "../moon/nextgen/pdk" } extism-pdk = "1.0.0" +moon_pdk = "0.0.1" +moon_pdk_test_utils = "0.0.1" +starbase_sandbox = "0.1.13" + +[profile.release] +codegen-units = 1 +debug = false +lto = true +opt-level = "s" +panic = "abort" diff --git a/crates/dlu/Cargo.toml b/crates/common/Cargo.toml similarity index 75% rename from crates/dlu/Cargo.toml rename to crates/common/Cargo.toml index 6143159..0bfa37d 100644 --- a/crates/dlu/Cargo.toml +++ b/crates/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -name = "moon_extension_dlu" +name = "moon_extension_common" version = "0.1.0" edition = "2021" +license = "MIT" publish = false [dependencies] diff --git a/crates/common/src/download.rs b/crates/common/src/download.rs new file mode 100644 index 0000000..a041f96 --- /dev/null +++ b/crates/common/src/download.rs @@ -0,0 +1,30 @@ +use extism_pdk::debug; +use moon_pdk::{fetch_url_bytes, AnyResult, VirtualPath}; +use std::fs; + +pub fn download_from_url, P: AsRef>( + src_url: U, + dst_dir: P, + file_name: Option<&str>, +) -> AnyResult { + let url = src_url.as_ref(); + let dir = dst_dir.as_ref(); + + debug!("Downloading file from {}", url); + + // Extract the file name from the URL + let file_name = file_name.unwrap_or_else(|| &url[url.rfind('/').unwrap() + 1..]); + + // Fetch the bytes of the URL + let bytes = fetch_url_bytes(url)?; + + // Write the to the provided file + let file = dir.join(file_name); + + fs::create_dir_all(dir)?; + fs::write(&file, bytes)?; + + debug!("Downloaded to {}", file.real_path().display()); + + Ok(file) +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs new file mode 100644 index 0000000..674b799 --- /dev/null +++ b/crates/common/src/lib.rs @@ -0,0 +1 @@ +pub mod download; diff --git a/crates/dlu/src/lib.rs b/crates/dlu/src/lib.rs deleted file mode 100644 index ef10715..0000000 --- a/crates/dlu/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -use extism_pdk::*; -use moon_pdk::extension::*; diff --git a/crates/download/CHANGELOG.md b/crates/download/CHANGELOG.md new file mode 100644 index 0000000..1fa16cb --- /dev/null +++ b/crates/download/CHANGELOG.md @@ -0,0 +1,5 @@ +## 0.1.0 + +#### 🚀 Updates + +- Initial release! diff --git a/crates/download/Cargo.toml b/crates/download/Cargo.toml new file mode 100644 index 0000000..5fe7c49 --- /dev/null +++ b/crates/download/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "moon_extension_download" +version = "0.0.1" +edition = "2021" +license = "MIT" +publish = false + +[lib] +crate-type = ['cdylib'] + +[dependencies] +moon_extension_common = { path = "../common" } +extism-pdk = { workspace = true } +moon_pdk = { workspace = true } + +[dev-dependencies] +moon_pdk_test_utils = { workspace = true } +starbase_sandbox = { workspace = true } + +[features] +default = ["wasm"] +wasm = [] diff --git a/crates/download/src/download_ext.rs b/crates/download/src/download_ext.rs new file mode 100644 index 0000000..a0842dc --- /dev/null +++ b/crates/download/src/download_ext.rs @@ -0,0 +1,68 @@ +use extism_pdk::*; +use moon_extension_common::download::download_from_url; +use moon_pdk::{ + anyhow, args::*, extension::*, host_log, plugin_err, virtual_path, HostLogInput, HostLogTarget, + VirtualPath, +}; + +#[host_fn] +extern "ExtismHost" { + fn host_log(input: Json); + fn to_virtual_path(path: String) -> String; +} + +#[derive(Args)] +pub struct DownloadExtensionArgs { + #[arg(long, short = 'u', required = true)] + pub url: String, + + #[arg(long, short = 'd')] + pub dest: Option, + + #[arg(long)] + pub name: Option, +} + +#[plugin_fn] +pub fn execute_extension(Json(input): Json) -> FnResult<()> { + let args = parse_args::(&input.args)?; + + if !args.url.starts_with("http") { + return Err(plugin_err!("A valid URL is required for downloading.")); + } + + // Determine destination directory + debug!("Determining destination directory"); + + let dest_dir = virtual_path!( + buf, + input + .context + .get_absolute_path(args.dest.as_deref().unwrap_or_default()) + ); + + if dest_dir.exists() && dest_dir.is_file() { + return Err(plugin_err!( + "Destination {} must be a directory, found a file.", + dest_dir.real_path().display(), + )); + } + + debug!( + "Destination {} will be used", + dest_dir.real_path().display(), + ); + + // Attempt to download the file + host_log!(stdout, "Downloading {}", args.url); + + let dest_file = download_from_url(&args.url, &dest_dir, args.name.as_deref())?; + + host_log!( + stdout, + "Downloaded to {}", + dest_file.real_path().display() + ); + + Ok(()) +} diff --git a/crates/download/src/lib.rs b/crates/download/src/lib.rs new file mode 100644 index 0000000..2595763 --- /dev/null +++ b/crates/download/src/lib.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "wasm")] +mod download_ext; + +#[cfg(feature = "wasm")] +pub use download_ext::*; diff --git a/crates/download/tests/download_test.rs b/crates/download/tests/download_test.rs new file mode 100644 index 0000000..ea4d411 --- /dev/null +++ b/crates/download/tests/download_test.rs @@ -0,0 +1,109 @@ +use moon_pdk_test_utils::{create_extension, ExecuteExtensionInput}; +use starbase_sandbox::create_empty_sandbox; +use std::fs; + +mod download { + use super::*; + + #[test] + #[should_panic(expected = "the following required arguments were not provided")] + fn errors_if_no_args() { + let sandbox = create_empty_sandbox(); + let plugin = create_extension("test", sandbox.path()); + + plugin.execute_extension(ExecuteExtensionInput { + args: vec![], + context: plugin.create_context(sandbox.path()), + }); + } + + #[test] + #[should_panic(expected = "A valid URL is required for downloading.")] + fn errors_if_not_a_url() { + let sandbox = create_empty_sandbox(); + let plugin = create_extension("test", sandbox.path()); + + plugin.execute_extension(ExecuteExtensionInput { + args: vec!["--url".into(), "invalid".into()], + context: plugin.create_context(sandbox.path()), + }); + } + + #[test] + #[should_panic(expected = "must be a directory, found a file")] + fn errors_if_dest_is_a_file() { + let sandbox = create_empty_sandbox(); + let plugin = create_extension("test", sandbox.path()); + + sandbox.create_file("dest", "file"); + + plugin.execute_extension(ExecuteExtensionInput { + args: vec![ + "--url".into(), + "https://raw.githubusercontent.com/moonrepo/moon/master/README.md".into(), + "--dest".into(), + "./dest".into(), + ], + context: plugin.create_context(sandbox.path()), + }); + } + + #[test] + fn downloads_file() { + let sandbox = create_empty_sandbox(); + let plugin = create_extension("test", sandbox.path()); + + plugin.execute_extension(ExecuteExtensionInput { + args: vec![ + "--url".into(), + "https://raw.githubusercontent.com/moonrepo/moon/master/README.md".into(), + "--dest".into(), + ".".into(), + ], + context: plugin.create_context(sandbox.path()), + }); + + let file = sandbox.path().join("README.md"); + + assert!(file.exists()); + assert_eq!(fs::metadata(file).unwrap().len(), 4013); + } + + #[test] + fn downloads_file_to_subdir() { + let sandbox = create_empty_sandbox(); + let plugin = create_extension("test", sandbox.path()); + + plugin.execute_extension(ExecuteExtensionInput { + args: vec![ + "--url".into(), + "https://raw.githubusercontent.com/moonrepo/moon/master/README.md".into(), + "--dest".into(), + "./sub/dir".into(), + ], + context: plugin.create_context(sandbox.path()), + }); + + assert!(sandbox.path().join("sub/dir/README.md").exists()); + } + + #[test] + fn downloads_file_with_custom_name() { + let sandbox = create_empty_sandbox(); + let plugin = create_extension("test", sandbox.path()); + + plugin.execute_extension(ExecuteExtensionInput { + args: vec![ + "--url".into(), + "https://raw.githubusercontent.com/moonrepo/moon/master/README.md".into(), + "--dest".into(), + "./sub/dir".into(), + "--name".into(), + "moon.md".into(), + ], + context: plugin.create_context(sandbox.path()), + }); + + assert!(sandbox.path().join("sub/dir/moon.md").exists()); + } +} diff --git a/crates/unpack/CHANGELOG.md b/crates/unpack/CHANGELOG.md new file mode 100644 index 0000000..1fa16cb --- /dev/null +++ b/crates/unpack/CHANGELOG.md @@ -0,0 +1,5 @@ +## 0.1.0 + +#### 🚀 Updates + +- Initial release! diff --git a/crates/unpack/Cargo.toml b/crates/unpack/Cargo.toml new file mode 100644 index 0000000..29486ce --- /dev/null +++ b/crates/unpack/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "moon_extension_unpack" +version = "0.0.1" +edition = "2021" +license = "MIT" +publish = false + +[lib] +crate-type = ['cdylib'] + +[dependencies] +moon_extension_common = { path = "../common" } +extism-pdk = { workspace = true } +moon_pdk = { workspace = true } +starbase_archive = { version = "0.2.5", default-features = false, features = [ + "tar-gz", + "zip", +] } + +[dev-dependencies] +moon_pdk_test_utils = { workspace = true } +starbase_sandbox = { workspace = true } + +[features] +default = ["wasm"] +wasm = [] diff --git a/crates/unpack/src/lib.rs b/crates/unpack/src/lib.rs new file mode 100644 index 0000000..e8b7e0c --- /dev/null +++ b/crates/unpack/src/lib.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "wasm")] +mod unpack_ext; + +#[cfg(feature = "wasm")] +pub use unpack_ext::*; diff --git a/crates/unpack/src/unpack_ext.rs b/crates/unpack/src/unpack_ext.rs new file mode 100644 index 0000000..f2ac374 --- /dev/null +++ b/crates/unpack/src/unpack_ext.rs @@ -0,0 +1,119 @@ +use extism_pdk::*; +use moon_extension_common::download::download_from_url; +use moon_pdk::{ + anyhow, args::*, extension::*, host_log, plugin_err, virtual_path, HostLogInput, HostLogTarget, + VirtualPath, +}; +use starbase_archive::Archiver; +use std::fs; + +#[host_fn] +extern "ExtismHost" { + fn host_log(input: Json); + fn to_virtual_path(path: String) -> String; +} + +#[derive(Args)] +pub struct UnpackExtensionArgs { + #[arg(long, short = 's', required = true)] + pub src: String, + + #[arg(long, short = 'd')] + pub dest: Option, + + #[arg(long)] + pub prefix: Option, +} + +#[plugin_fn] +pub fn execute_extension(Json(input): Json) -> FnResult<()> { + let args = parse_args::(&input.args)?; + + // Determine the correct input. If the input is a URL, attempt to download + // the file, otherwise use the file directly (if within our whitelist). + let src_file = if args.src.starts_with("http") { + debug!("Received a URL as the input source"); + + download_from_url(&args.src, virtual_path!("/moon/temp"), None)? + } else { + debug!( + "Converting source {} to an absolute virtual path", + args.src + ); + + virtual_path!(buf, input.context.get_absolute_path(args.src)) + }; + + if !src_file + .extension() + .is_some_and(|ext| ext == "tar" || ext == "tgz" || ext == "gz" || ext == "zip") + { + return Err(plugin_err!( + "Invalid source, only .tar, .tar.gz, and .zip archives are supported." + )); + } + + if !src_file.exists() || !src_file.is_file() { + return Err(plugin_err!( + "Source {} must be a valid file.", + src_file.real_path().display(), + )); + } + + host_log!( + stdout, + "Opening archive {}", + src_file.real_path().display() + ); + + // Convert the provided output into a virtual file path. + let dest_dir = virtual_path!( + buf, + input + .context + .get_absolute_path(args.dest.as_deref().unwrap_or_default()) + ); + + if dest_dir.exists() && dest_dir.is_file() { + return Err(plugin_err!( + "Destination {} must be a directory, found a file.", + dest_dir.real_path().display(), + )); + } + + fs::create_dir_all(&dest_dir)?; + + host_log!( + stdout, + "Unpacking archive to {}", + dest_dir.real_path().display() + ); + + // Attempt to unpack the archive! + let mut archive = Archiver::new(&dest_dir, &src_file); + + // Diff against all files in the output dir + archive.add_source_glob("**/*"); + + // Remove the prefix from unpacked files + if let Some(prefix) = &args.prefix { + archive.set_prefix(prefix); + } + + // Unpack the files + if let Err(error) = archive.unpack_from_ext() { + let mut message = error.to_string(); + + // Miette hides the real error + if let Some(source) = error.source() { + message.push(' '); + message.push_str(&source.to_string()); + } + + return Err(plugin_err!("{message}")); + }; + + host_log!(stdout, "Unpacked archive!"); + + Ok(()) +} diff --git a/crates/unpack/tests/__fixtures__/tar/archive.tar b/crates/unpack/tests/__fixtures__/tar/archive.tar new file mode 100644 index 0000000000000000000000000000000000000000..ba692485e0eb1ffa62e9c71071e4070d34549e2e GIT binary patch literal 2560 zcmeH^Q4Yc&42F636!ijIVRXBQ7g!VmiTVI&JpMYi2bY+=kr<^RA*D_E`~6LO@u3w( z0}%%z5cAzHH!7WTe#!w-6KO@HPy$M&2nA44RqOCaOk-eaH-9y8=R9mv7Q5m666gdcFAl+5ZmT{3rIm8x#q^GIF+4|J?tR^AYBAhvW>5;q3g= T91mMaHLCy)CHjN_{Fv{x?x9* zkr$G`U=sfDWOwqR6IDUPkr|d8)B z%-X{Jw#O@f1iARPd3l*0sm>euKL)J}{`+_Toc;GBu>h=7=aWD8e{ep+^IRc015-FU Ye>eNxw|g}jjYgv`Gh=8o#{dcd0Fwbv>;M1& literal 0 HcmV?d00001 diff --git a/crates/unpack/tests/__fixtures__/zip/archive.zip b/crates/unpack/tests/__fixtures__/zip/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..5929fb3d8db48f3475c492cd9046d43cae4eeb81 GIT binary patch literal 373 zcmWIWW@Zs#0D+EBiwH0SO0X~}Fr;J_>4%2!GO#yBZA@(gVh~+g!Og(P@|BT+0jx6s zYyt-Z2SXQ<2@*h4c%UYvW#**nl~j~~O=<<2&;`UG8qFl`lP7e-7#7;JPU;D1J+kOY z&zVgju6hgs-s~LTzsxoh2O0@7Ho%*aNrV}