From 294104df2b62d6b0841bba4b6cb323e974d4c851 Mon Sep 17 00:00:00 2001 From: Mendy Berger <12537668+MendyBerger@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:10:42 -0400 Subject: [PATCH 1/4] Codegen update to #218 (code only) --- bundled/componentize_py_exports.py | 40 + bundled/poll_loop.py | 13 +- runtime/src/lib.rs | 71 +- src/command.rs | 124 ++-- src/lib.rs | 249 +++++-- src/python.rs | 34 +- src/summary.rs | 1085 +++++++++++++++++++++------- wit/init.wit | 10 +- 8 files changed, 1216 insertions(+), 410 deletions(-) create mode 100644 bundled/componentize_py_exports.py diff --git a/bundled/componentize_py_exports.py b/bundled/componentize_py_exports.py new file mode 100644 index 00000000..c45cd401 --- /dev/null +++ b/bundled/componentize_py_exports.py @@ -0,0 +1,40 @@ +"""Registry of the implementations an app provides for its WIT exports. + +The bindings generated by `componentize-py` include a decorator for each +exported interface, resource, and world, and applying one of those decorators +registers the decorated class here. `componentize-py` reads this registry when +pre-initializing the component in order to connect each WIT export to the code +implementing it. + +Application code should not use this module directly; use the generated +decorators instead, e.g.:: + + from wit.exports.wasi.http_v0_2 import incoming_handler + + @incoming_handler.guest + class MyHandler: + ... +""" + +from typing import Any, Callable, Dict, TypeVar + +T = TypeVar("T") + +# Export key (see `scope` in `init.wit`) -> implementing class. +EXPORTS: Dict[str, Any] = {} + + +def register(key: str) -> Callable[[T], T]: + """Return a decorator which registers the class it's applied to as the + implementation of the WIT export named by `key`.""" + + def decorate(value: T) -> T: + if key in EXPORTS: + raise RuntimeError( + f"multiple implementations registered for `{key}`; " + f"exactly one is expected" + ) + EXPORTS[key] = value + return value + + return decorate diff --git a/bundled/poll_loop.py b/bundled/poll_loop.py index 504c958a..5c5d5360 100644 --- a/bundled/poll_loop.py +++ b/bundled/poll_loop.py @@ -11,15 +11,20 @@ import subprocess from componentize_py_types import Ok, Err -from wit_world.imports import types, streams, poll, outgoing_handler -from wit_world.imports.types import ( + +# Placeholders replaced at build time; see `HELPER_INTERFACES` in summary.rs. +import WASI_HTTP_TYPES_MODULE as types +import WASI_HTTP_OUTGOING_HANDLER_MODULE as outgoing_handler +import WASI_IO_STREAMS_MODULE as streams +import WASI_IO_POLL_MODULE as poll +from WASI_HTTP_TYPES_MODULE import ( IncomingBody, OutgoingBody, OutgoingRequest, IncomingResponse, ) -from wit_world.imports.streams import StreamError_Closed, InputStream -from wit_world.imports.poll import Pollable +from WASI_IO_STREAMS_MODULE import StreamError_Closed, InputStream +from WASI_IO_POLL_MODULE import Pollable from typing import Optional, cast # Maximum number of bytes to read at a time diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 4eb7c4e3..be8cd710 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -908,10 +908,30 @@ fn do_init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), St Python::initialize(); let init = |py: Python| { - let app = py.import(app_name.as_str())?; + // Importing the app runs its decorators, filling `componentize_py_exports.EXPORTS`. + py.import(app_name.as_str())?; + + let registry = py.import("componentize_py_exports")?.getattr("EXPORTS")?; + + let implementation = |scope: &str| { + registry.get_item(scope).map_err(|error| { + if error.is_instance_of::(py) { + PyAssertionError::new_err(format!( + "no implementation registered for `{scope}`; please apply the \ + corresponding decorator from the generated bindings to the class \ + implementing it" + )) + } else { + error + } + }) + }; STUB_WASI.set(stub_wasi).unwrap(); + // One implementation instance per interface, shared by its functions. + let mut instances = std::collections::HashMap::>::new(); + EXPORTS .set( symbols @@ -920,35 +940,32 @@ fn do_init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), St .map(|export| { Ok(Export { kind: match &export.kind { - FunctionExportKind::Freestanding(exp::Function { - protocol, - name, - }) => ExportKind::Freestanding { - name: PyString::intern(py, name).into(), - instance: app.getattr(protocol.as_str())?.call0()?.into(), - }, - FunctionExportKind::Constructor(Constructor { - module, - protocol, - }) => ExportKind::Constructor( - py.import(module.as_str())? - .getattr(protocol.as_str())? - .into(), - ), + FunctionExportKind::Freestanding(exp::Function { scope, name }) => { + let instance = if let Some(instance) = instances.get(scope) { + instance.clone_ref(py) + } else { + let instance: Py = + implementation(scope)?.call0()?.into(); + instances.insert(scope.clone(), instance.clone_ref(py)); + instance + }; + ExportKind::Freestanding { + name: PyString::intern(py, name).into(), + instance, + } + } + FunctionExportKind::Constructor(Constructor { scope }) => { + ExportKind::Constructor(implementation(scope)?.into()) + } FunctionExportKind::Method(name) => { ExportKind::Method(PyString::intern(py, name).into()) } - FunctionExportKind::Static(Static { - module, - protocol, - name, - }) => ExportKind::Static { - name: PyString::intern(py, name).into(), - class: py - .import(module.as_str())? - .getattr(protocol.as_str())? - .into(), - }, + FunctionExportKind::Static(Static { scope, name }) => { + ExportKind::Static { + name: PyString::intern(py, name).into(), + class: implementation(scope)?.into(), + } + } }, return_style: export.return_style, }) diff --git a/src/command.rs b/src/command.rs index be886b69..8b018cb0 100644 --- a/src/command.rs +++ b/src/command.rs @@ -19,10 +19,25 @@ pub struct Options { #[command(flatten)] pub common: Common, + #[command(flatten)] + pub deprecated: Deprecated, + #[command(subcommand)] pub command: Command, } +/// Options kept only for back-compat; `run` folds them into `Common`. +#[derive(clap::Args, Clone, Debug)] +pub struct Deprecated { + /// Deprecated: renamed to `--bindings-module`. + #[arg(long, hide = true)] + pub world_module: Option, + + /// Deprecated and ignored: fully-qualified module names are always used. + #[arg(long, hide = true, action = clap::ArgAction::SetTrue)] + pub full_names: Option, +} + #[derive(clap::Args, Clone, Debug)] pub struct Common { /// Files or directories containing WIT document(s). @@ -59,40 +74,36 @@ pub struct Common { /// Specify names to use for imported interfaces. May be specified more /// than once. /// - /// By default, the python module name generated for a given interface will - /// be the snake-case form of the WIT interface name, possibly qualified - /// with the package name and namespace and/or version if that name would - /// otherwise clash with another interface. With this option, you may - /// override that name with your own, unique name. + /// By default, the Python module generated for a given interface is nested + /// according to the fully-qualified WIT interface name, i.e. the package + /// namespace, then the package name plus its (semver-canonical) version, + /// then the interface name (e.g. `wit.imports.wasi.http_v0_2.types` for + /// `wasi:http/types@0.2.0`). With this option, you may override that with + /// your own, unique name, which may itself be a dotted path (e.g. + /// `my_package.my_module`) and is used verbatim. Each component must be a + /// valid Python identifier. #[arg(long, value_parser = parse_key_value)] pub import_interface_name: Vec<(String, String)>, /// Specify names to use for exported interfaces. May be specified more /// than once. /// - /// By default, the python module name generated for a given interface will - /// be the snake-case form of the WIT interface name, possibly qualified - /// with the package name and namespace and/or version if that name would - /// otherwise clash with another interface. With this option, you may - /// override that name with your own, unique name. + /// By default, the Python module generated for a given interface is nested + /// according to the fully-qualified WIT interface name, i.e. the package + /// namespace, then the package name plus its (semver-canonical) version, + /// then the interface name (e.g. `wit.exports.wasi.http_v0_2.types` for + /// `wasi:http/types@0.2.0`). With this option, you may override that with + /// your own, unique name, which may itself be a dotted path (e.g. + /// `my_package.my_module`) and is used verbatim. Each component must be a + /// valid Python identifier. #[arg(long, value_parser = parse_key_value)] pub export_interface_name: Vec<(String, String)>, /// Optional name of top-level module to use for bindings. /// - /// If this is not specified, the module name will default to "wit_world". + /// If this is not specified, the module name will default to "wit". #[arg(long)] - pub world_module: Option, - - /// When generating Python module names, include the WIT package name and - /// version even if only one version of that package is referenced by the - /// specified world or only one package uses that name. - /// - /// By default, the package name and version will only be included in the - /// name if the world references more than one version of the WIT package or - /// the name is used by more than one package. - #[arg(long)] - pub full_names: bool, + pub bindings_module: Option, } #[derive(clap::Subcommand, Debug)] @@ -117,10 +128,9 @@ pub enum Command { pub struct Componentize { /// The name of a Python module containing the app to wrap. /// - /// Note that this should not match (any of) the world name(s) you are - /// targeting since `componentize-py` will generate code using those - /// name(s), and Python doesn't know how to load two top-level modules with - /// the same name. + /// Note that this should not match the bindings module name (`wit` by + /// default, see `--bindings-module`), since Python can't load two + /// top-level modules with the same name. pub app_name: String, /// Specify a directory containing the app and/or its dependencies. May be @@ -205,10 +215,22 @@ fn parse_key_value(s: &str) -> Result<(String, String), String> { } pub fn run + Clone, I: IntoIterator>(args: I) -> Result<()> { - let options = Options::parse_from(args); - match options.command { - Command::Componentize(opts) => componentize(options.common, opts), - Command::Bindings(opts) => generate_bindings(options.common, opts), + let Options { + mut common, + deprecated, + command, + } = Options::parse_from(args); + + common.bindings_module = crate::resolve_deprecated( + common.bindings_module, + deprecated.world_module, + deprecated.full_names, + common.quiet, + )?; + + match command { + Command::Componentize(opts) => componentize(common, opts), + Command::Bindings(opts) => generate_bindings(common, opts), } } @@ -226,7 +248,7 @@ fn generate_bindings(common: Common, bindings: Bindings) -> Result<()> { .map(|v| v.as_str()) .collect::>(), all_features: common.all_features, - world_module: common.world_module.as_deref(), + bindings_module: common.bindings_module.as_deref(), output_dir: &bindings.output_dir, import_interface_names: &common .import_interface_name @@ -238,7 +260,6 @@ fn generate_bindings(common: Common, bindings: Bindings) -> Result<()> { .iter() .map(|(a, b)| (a.as_str(), b.as_str())) .collect(), - full_names: common.full_names, } .generate() } @@ -269,7 +290,7 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> { .map(|v| v.as_str()) .collect::>(), all_features: common.all_features, - world_module: common.world_module.as_deref(), + bindings_module: common.bindings_module.as_deref(), python_path: &python_path.iter().map(|s| s.as_str()).collect::>(), module_worlds: &componentize .module_worlds @@ -293,7 +314,6 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> { .iter() .map(|(a, b)| (a.as_str(), b.as_str())) .collect(), - full_names: common.full_names, intersect_world: componentize.intersect_world.as_deref(), } .generate(), @@ -427,13 +447,12 @@ mod tests { let common = Common { wit_path: vec![wit.path().into()], world: Vec::new(), - world_module: Some("bindings".into()), + bindings_module: Some("bindings".into()), quiet: false, features: vec![], all_features: false, import_interface_name: Vec::new(), export_interface_name: Vec::new(), - full_names: false, }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -458,13 +477,12 @@ mod tests { let common = Common { wit_path: vec![wit.path().into()], world: Vec::new(), - world_module: Some("bindings".into()), + bindings_module: Some("bindings".into()), quiet: false, features: vec!["x".to_owned()], all_features: false, import_interface_name: Vec::new(), export_interface_name: Vec::new(), - full_names: false, }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -489,13 +507,12 @@ mod tests { let common = Common { wit_path: vec![wit.path().into()], world: Vec::new(), - world_module: Some("bindings".into()), + bindings_module: Some("bindings".into()), quiet: false, features: vec![], all_features: true, import_interface_name: Vec::new(), export_interface_name: Vec::new(), - full_names: false, }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -519,13 +536,12 @@ mod tests { let common = Common { wit_path: vec![wit.path().into()], world: Vec::new(), - world_module: Some("bindings".into()), + bindings_module: Some("bindings".into()), quiet: false, features: vec!["x".to_owned()], all_features: false, import_interface_name: Vec::new(), export_interface_name: Vec::new(), - full_names: false, }; let bindings = Bindings { output_dir: out_dir.path().into(), @@ -536,8 +552,10 @@ mod tests { r#" import bindings from bindings import x +from bindings.export import world_exports -class Bindings(bindings.Bindings): +@world_exports +class Bindings(bindings.WorldExports): def y(self) -> None: x() "#, @@ -582,14 +600,18 @@ world cli-world { &app_file, br#" import cli_world -from cli_world import exports -from cli_world.imports import cli_interface -from lib.wit.imports import lib_interface - -class CliWorld(cli_world.CliWorld): +from cli_world.export import world_exports +from cli_world.export.test import cli as export +from cli_world.exports.test import cli as exports +from cli_world.imports.test.cli import cli_interface +from lib.wit.imports.test.lib import lib_interface + +@world_exports +class CliWorld(cli_world.WorldExports): def foo(self) -> None: pass +@export.cli_interface class CliInterface(exports.CliInterface): def foo(self) -> None: lib_interface.foo() @@ -628,13 +650,12 @@ world lib-world { Common { wit_path: vec![lib_wit_dir.clone()], world: vec!["test:lib/lib-world".into()], - world_module: Some("lib.wit".into()), + bindings_module: Some("lib.wit".into()), quiet: false, features: Vec::new(), all_features: false, import_interface_name: Vec::new(), export_interface_name: Vec::new(), - full_names: false, }, Bindings { output_dir: lib_wit_dir, @@ -645,13 +666,12 @@ world lib-world { Common { wit_path: vec![cli_wit_file], world: vec!["test:cli/cli-world".into(), "test:lib/lib-world".into()], - world_module: Some("cli_world".into()), + bindings_module: Some("cli_world".into()), quiet: false, features: Vec::new(), all_features: false, import_interface_name: Vec::new(), export_interface_name: Vec::new(), - full_names: false, }, Componentize { app_name: "app".into(), diff --git a/src/lib.rs b/src/lib.rs index 6ccb33df..e1ad3a1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ #![deny(warnings)] use { - anyhow::{Context, Error, Result, anyhow, ensure}, + anyhow::{Context, Error, Result, anyhow, bail, ensure}, async_trait::async_trait, bytes::Bytes, component_init_transform::Invoker, @@ -18,7 +18,7 @@ use { path::{Path, PathBuf}, str, }, - summary::{Locations, Naming, Summary}, + summary::{Locations, Summary}, tar::Archive, wasm_encoder::{CustomSection, Section as _}, wasmtime::{ @@ -51,10 +51,8 @@ mod util; const DEBUG_PYTHON_BINDINGS: bool = false; -/// The default name of the Python module containing code generated from the -/// specified WIT world. This may be overriden programatically or via the CLI -/// using the `--world-module` option. -static DEFAULT_WORLD_MODULE: &str = "wit_world"; +/// Default bindings module name, overridable via `--bindings-module`. +static DEFAULT_BINDINGS_MODULE: &str = "wit"; wasmtime::component::bindgen!({ path: "wit", @@ -90,8 +88,7 @@ struct RawComponentizePyConfig { import_interface_names: HashMap, #[serde(default)] export_interface_names: HashMap, - #[serde(default)] - full_names: bool, + full_names: Option, } #[derive(Debug)] @@ -100,13 +97,14 @@ struct ComponentizePyConfig { wit_directory: Option, import_interface_names: HashMap, export_interface_names: HashMap, - full_names: bool, } impl TryFrom<(&Path, RawComponentizePyConfig)> for ComponentizePyConfig { type Error = Error; fn try_from((path, raw): (&Path, RawComponentizePyConfig)) -> Result { + warn_full_names_deprecated(raw.full_names, Some(path)); + let base = path.canonicalize()?; let convert = |p| { // Ensure this is a relative path under `base`: @@ -121,7 +119,6 @@ impl TryFrom<(&Path, RawComponentizePyConfig)> for ComponentizePyConfig { wit_directory: raw.wit_directory.map(convert).transpose()?, import_interface_names: raw.import_interface_names, export_interface_names: raw.export_interface_names, - full_names: raw.full_names, }) } } @@ -182,11 +179,10 @@ pub struct BindingsGenerator<'a> { pub worlds: &'a [&'a str], pub features: &'a [&'a str], pub all_features: bool, - pub world_module: Option<&'a str>, + pub bindings_module: Option<&'a str>, pub output_dir: &'a Path, pub import_interface_names: &'a HashMap<&'a str, &'a str>, pub export_interface_names: &'a HashMap<&'a str, &'a str>, - pub full_names: bool, } impl BindingsGenerator<'_> { @@ -234,35 +230,170 @@ impl BindingsGenerator<'_> { let stream_and_future_indexes = &HashMap::new(); let summary = Summary::try_new( &resolve, - &iter::once((world, Naming::from_full(self.full_names))).collect(), + &iter::once(world).collect(), self.import_interface_names, self.export_interface_names, import_function_indexes, export_function_indexes, stream_and_future_indexes, )?; - let world_module = self.world_module.unwrap_or(DEFAULT_WORLD_MODULE); - let world_dir = self.output_dir.join(world_module.replace('.', "/")); + let bindings_module = self.bindings_module.unwrap_or(DEFAULT_BINDINGS_MODULE); + validate_bindings_module(bindings_module)?; + let world_dir = self.output_dir.join(bindings_module.replace('.', "/")); + // A `wit/` WIT-source directory clashes with the default module name. + if world_dir.is_dir() { + if contains_wit_files(&world_dir)? { + bail!( + "refusing to write bindings into {}, which contains WIT source files; \ + specify a different output directory or `--bindings-module`", + world_dir.display() + ); + } + fs::remove_dir_all(&world_dir)?; + } fs::create_dir_all(&world_dir)?; - summary.generate_code( - &world_dir, - world, - world_module, - &mut Locations::default(), - true, - )?; + create_module_ancestors(self.output_dir, bindings_module)?; + let mut locations = Locations::default(); + summary.generate_code(&world_dir, world, bindings_module, &mut locations, true)?; + let helper_module_paths = summary.helper_module_paths(&locations); - Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( + let mut archive = Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), "/bundled.tar.zst" - ))))?) - .unpack(self.output_dir) - .unwrap(); + ))))?); + + // Leave anything else in the output dir alone (e.g. the app itself) + for entry in archive.entries()? { + let mut entry = entry?; + let relative = entry.path()?.into_owned(); + ensure!( + relative + .components() + .all(|c| matches!(c, std::path::Component::Normal(_))), + "invalid path in bundled archive: {}", + relative.display() + ); + let path = self.output_dir.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + entry.unpack(&path)?; + if path.is_file() { + resolve_helper_placeholders(&path, &helper_module_paths)?; + } + } + + Ok(()) + } +} + +/// Warn about the deprecated `full_names` option (CLI, TOML, or Python API). +pub(crate) fn warn_full_names_deprecated(full_names: Option, source: Option<&Path>) { + if full_names == Some(true) { + let source = source + .map(|path| format!(" in {}", path.display())) + .unwrap_or_default(); + eprintln!( + "warning: `full_names`{source} is deprecated and has no effect; fully-qualified \ + module names are always used" + ); + } +} + +/// Resolve the deprecated `world_module` spelling of `bindings_module`, warning +/// about it and about `full_names`. +pub(crate) fn resolve_deprecated( + bindings_module: Option, + world_module: Option, + full_names: Option, + quiet: bool, +) -> Result> { + if !quiet { + warn_full_names_deprecated(full_names, None); + if world_module.is_some() { + eprintln!("warning: `world_module` is deprecated; use `bindings_module`"); + } + } + ensure!( + !(bindings_module.is_some() && world_module.is_some()), + "`bindings_module` and its deprecated spelling `world_module` are mutually exclusive" + ); + + Ok(bindings_module.or(world_module)) +} + +fn contains_wit_files(dir: &Path) -> Result { + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + if contains_wit_files(&path)? { + return Ok(true); + } + } else if path.extension().is_some_and(|ext| ext == "wit") { + return Ok(true); + } + } + + Ok(false) +} +/// Reject `--bindings-module` values which are not importable Python paths. +fn validate_bindings_module(module: &str) -> Result<()> { + if module + .split('.') + .all(|c| summary::is_python_identifier(c) && !summary::is_python_keyword(c)) + { Ok(()) + } else { + bail!("`{module}` is not a valid Python module path") } } +/// Make each ancestor of a dotted bindings module a regular package. +fn create_module_ancestors(root: &Path, module: &str) -> Result<()> { + let components = module.split('.').collect::>(); + let mut dir = root.to_owned(); + for component in &components[..components.len() - 1] { + dir = dir.join(component); + fs::create_dir_all(&dir)?; + let init = dir.join("__init__.py"); + if !init.exists() { + fs::write(init, "")?; + } + } + + Ok(()) +} + +/// Replace each helper placeholder with the module path generated for it; an +/// unimported interface has no entry and keeps its placeholder. +fn resolve_helper_placeholders(path: &Path, replacements: &[(&str, String)]) -> Result<()> { + if path.is_dir() { + for entry in fs::read_dir(path)? { + resolve_helper_placeholders(&entry?.path(), replacements)?; + } + } else { + // Skip binary files, and leave files without a placeholder untouched. + let bytes = fs::read(path)?; + if let Ok(text) = str::from_utf8(&bytes) + && let Some(replaced) = replacements + .iter() + .filter(|(placeholder, _)| text.contains(placeholder)) + .fold(None, |replaced: Option, (placeholder, module)| { + Some( + replaced + .unwrap_or_else(|| text.to_owned()) + .replace(placeholder, module), + ) + }) + { + fs::write(path, replaced)?; + } + } + + Ok(()) +} + pub type AddToLinker<'a> = Option<&'a dyn Fn(&mut Linker) -> Result<()>>; pub struct ComponentGenerator<'a> { @@ -270,7 +401,7 @@ pub struct ComponentGenerator<'a> { pub worlds: &'a [&'a str], pub features: &'a [&'a str], pub all_features: bool, - pub world_module: Option<&'a str>, + pub bindings_module: Option<&'a str>, pub python_path: &'a [&'a str], pub module_worlds: &'a [(&'a str, &'a [&'a str])], pub app_name: &'a str, @@ -279,7 +410,6 @@ pub struct ComponentGenerator<'a> { pub stub_wasi: bool, pub import_interface_names: &'a HashMap<&'a str, &'a str>, pub export_interface_names: &'a HashMap<&'a str, &'a str>, - pub full_names: bool, pub intersect_world: Option<&'a str>, } @@ -465,16 +595,12 @@ impl ComponentGenerator<'_> { let mut all_worlds = worlds .iter() .copied() - .map(|world| (world, Naming::from_full(self.full_names))) - .chain(configs.values().flat_map(|(config, worlds)| { - worlds.iter().copied().map(|world| { - ( - world, - Naming::from_full(config.config.full_names || self.full_names), - ) - }) - })) - .collect::>(); + .chain( + configs + .values() + .flat_map(|(_, worlds)| worlds.iter().copied()), + ) + .collect::>(); if all_worlds.is_empty() { // No worlds specified; pick the default one, if available: @@ -484,7 +610,7 @@ impl ComponentGenerator<'_> { intersect_world(&mut resolve, intersector, world); } - all_worlds.insert(world, Naming::from_full(self.full_names)); + all_worlds.insert(world); } // Now that we've parsed all known WIT files and resolved all relevant @@ -514,7 +640,7 @@ impl ComponentGenerator<'_> { let world = unioned( &mut resolve, - &all_worlds.keys().copied().collect::>(), + &all_worlds.iter().copied().collect::>(), )? .unwrap(); @@ -522,7 +648,7 @@ impl ComponentGenerator<'_> { // each module, union the ones covered by the module into a single // world. - let mut worlds_to_generate = all_worlds.keys().copied().collect::>(); + let mut worlds_to_generate = all_worlds.clone(); let configs = configs .iter() @@ -720,6 +846,8 @@ impl ComponentGenerator<'_> { .collect::>>()?; let binding_module = paths.first().unwrap().1.replace('/', "."); + validate_bindings_module(&binding_module) + .with_context(|| format!("in `bindings` for {}", config.path.display()))?; let world_dir = tempfile::tempdir()?; @@ -742,36 +870,23 @@ impl ComponentGenerator<'_> { // Here we generate code for any worlds not covered by any of the Python // modules we visited above. + let module = self.bindings_module.unwrap_or(DEFAULT_BINDINGS_MODULE); + validate_bindings_module(module)?; if let Some(world) = world_to_generate { - let module = self.world_module.unwrap_or(DEFAULT_WORLD_MODULE); let world_dir = tempfile::tempdir()?; - let module_path = world_dir.path().join(module); + let module_path = world_dir.path().join(module.replace('.', "/")); fs::create_dir_all(&module_path)?; + create_module_ancestors(world_dir.path(), module)?; summary.generate_code(&module_path, world, module, &mut locations, false)?; world_dir_mounts.push((vec!["world".to_owned()], world_dir)); - - // The helper utilities are hard-coded to assume the world module is - // named `wit_world`. Here we replace that with the actual world module - // name. - fn replace(path: &Path, pattern: &str, replacement: &str) -> Result<()> { - if path.is_dir() { - for entry in fs::read_dir(path)? { - replace(&entry?.path(), pattern, replacement)?; - } - } else { - fs::write( - path, - fs::read_to_string(path)? - .replace(pattern, replacement) - .as_bytes(), - )?; - } - - Ok(()) - } - replace(embedded_helper_utils.path(), "wit_world", module)?; }; + // One shared mount, but each interface has exactly one owning module. + resolve_helper_placeholders( + embedded_helper_utils.path(), + &summary.helper_module_paths(&locations), + )?; + for (mounts, world_dir) in world_dir_mounts.iter() { for mount in mounts { if DEBUG_PYTHON_BINDINGS { @@ -838,11 +953,7 @@ impl ComponentGenerator<'_> { async move { let component = &Component::new(&engine, instrumented)?; if !added_to_linker { - add_wasi_and_stubs( - &resolve, - &all_worlds.keys().copied().collect::>(), - &mut linker, - )?; + add_wasi_and_stubs(&resolve, &all_worlds, &mut linker)?; } let pre = InitPre::new(linker.instantiate_pre(component)?)?; diff --git a/src/python.rs b/src/python.rs index ba096855..9beca41a 100644 --- a/src/python.rs +++ b/src/python.rs @@ -18,13 +18,13 @@ use { #[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "componentize")] -#[pyo3(signature = (wit_path, worlds, features, all_features, world_module, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names, full_names, intersect_world))] +#[pyo3(signature = (wit_path, worlds, features, all_features, bindings_module, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names, full_names = None, intersect_world = None, world_module = None))] fn python_componentize( wit_path: Vec, worlds: Vec, features: Vec, all_features: bool, - world_module: Option<&str>, + bindings_module: Option<&str>, python_path: Vec, module_worlds: Vec<(PyBackedStr, Vec)>, app_name: &str, @@ -32,9 +32,17 @@ fn python_componentize( stub_wasi: bool, import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, - full_names: bool, + full_names: Option, intersect_world: Option<&str>, + world_module: Option<&str>, ) -> PyResult<()> { + let bindings_module = crate::resolve_deprecated( + bindings_module.map(str::to_owned), + world_module.map(str::to_owned), + full_names, + false, + ) + .map_err(|e| PyAssertionError::new_err(format!("{e:?}")))?; (|| { Runtime::new()?.block_on( ComponentGenerator { @@ -42,7 +50,7 @@ fn python_componentize( worlds: &worlds.iter().map(|v| v.as_str()).collect::>(), features: &features.iter().map(|v| v.as_str()).collect::>(), all_features, - world_module, + bindings_module: bindings_module.as_deref(), python_path: &python_path.iter().map(|s| s.as_ref()).collect::>(), module_worlds: &module_worlds .iter() @@ -63,7 +71,6 @@ fn python_componentize( .iter() .map(|(a, b)| (a.as_ref(), b.as_ref())) .collect(), - full_names, intersect_world, } .generate(), @@ -75,24 +82,32 @@ fn python_componentize( #[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "generate_bindings")] -#[pyo3(signature = (wit_path, worlds, features, all_features, world_module, output_dir, import_interface_names, export_interface_names, full_names))] +#[pyo3(signature = (wit_path, worlds, features, all_features, bindings_module, output_dir, import_interface_names, export_interface_names, full_names = None, world_module = None))] fn python_generate_bindings( wit_path: Vec, worlds: Vec, features: Vec, all_features: bool, - world_module: Option<&str>, + bindings_module: Option<&str>, output_dir: PathBuf, import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, - full_names: bool, + full_names: Option, + world_module: Option<&str>, ) -> PyResult<()> { + let bindings_module = crate::resolve_deprecated( + bindings_module.map(str::to_owned), + world_module.map(str::to_owned), + full_names, + false, + ) + .map_err(|e| PyAssertionError::new_err(format!("{e:?}")))?; BindingsGenerator { wit_paths: &wit_path.iter().map(|v| v.as_path()).collect::>(), worlds: &worlds.iter().map(|v| v.as_str()).collect::>(), features: &features.iter().map(|v| v.as_str()).collect::>(), all_features, - world_module, + bindings_module: bindings_module.as_deref(), output_dir: &output_dir, import_interface_names: &import_interface_names .iter() @@ -102,7 +117,6 @@ fn python_generate_bindings( .iter() .map(|(a, b)| (a.as_ref(), b.as_ref())) .collect(), - full_names, } .generate() .map_err(|e| PyAssertionError::new_err(format!("{e:?}"))) diff --git a/src/summary.rs b/src/summary.rs index 5fbe884c..9b6663c4 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -16,7 +16,7 @@ use { fs::{self, File}, io::Write as _, iter, - path::Path, + path::{Path, PathBuf}, str, }, wit_bindgen_core::Types, @@ -31,16 +31,109 @@ const NOT_IMPLEMENTED: &str = "raise NotImplementedError"; const ASYNC_START_PREFIX: &str = "_async_start_"; -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum Naming { - Short, - Full, +/// World-level exports ABC; snake-cased, also their `guest` decorator's bound. +const WORLD_EXPORTS_CLASS: &str = "WorldExports"; + +const FILE_HEADER: &str = "# This file is automatically generated by componentize-py +# It is not intended for manual editing. +"; + +/// Interfaces the bundled helpers import, and the placeholder each stands +/// behind, since hand-written helpers can't spell computed module paths. +pub const HELPER_INTERFACES: &[(&str, HelperInterface)] = &[ + ( + "WASI_HTTP_TYPES_MODULE", + HelperInterface { + package: "http", + interface: "types", + }, + ), + ( + "WASI_HTTP_OUTGOING_HANDLER_MODULE", + HelperInterface { + package: "http", + interface: "outgoing-handler", + }, + ), + ( + "WASI_IO_STREAMS_MODULE", + HelperInterface { + package: "io", + interface: "streams", + }, + ), + ( + "WASI_IO_POLL_MODULE", + HelperInterface { + package: "io", + interface: "poll", + }, + ), +]; + +/// A `wasi:{package}/{interface}@0.2.x` interface the bundled helpers import. +pub struct HelperInterface { + package: &'static str, + interface: &'static str, } -impl Naming { - pub fn from_full(full: bool) -> Self { - if full { Self::Full } else { Self::Short } +/// Registry key for an exported resource; documented on `scope` in `init.wit`. +fn resource_scope(scope: &str, resource: &str) -> String { + format!("{scope}#{resource}") +} + +/// Python module for a WIT interface, e.g. `imports.wasi.http_v0_2.types`. +struct InterfaceName { + /// Snake-cased path components, relative to `imports`/`exports`. + path: Vec, + /// Unique flat alias (e.g. `wasi_http_v0_2_types`) used by generated code. + flat: String, +} + +impl InterfaceName { + fn new(path: impl IntoIterator) -> Self { + let path = path + .into_iter() + .map(|c| c.to_snake_case().escape()) + .collect::>(); + let flat = path.join("_"); + + Self { path, flat } + } + + /// Accepts a plain name or a dotted path, used verbatim (callers validate). + fn from_override(name: &str) -> Self { + let path = name.split('.').map(str::to_owned).collect::>(); + let flat = path.join("_"); + + Self { path, flat } } + + fn dotted(&self) -> String { + self.path.join(".") + } + + fn parent(&self) -> Option { + (self.path.len() > 1).then(|| self.path[..self.path.len() - 1].join(".")) + } + + fn leaf(&self) -> &str { + self.path.last().unwrap() + } +} + +struct NamedInterface { + name: InterfaceName, + key: String, + overridden: bool, +} + +pub fn is_python_identifier(s: &str) -> bool { + let mut chars = s.chars(); + chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') } #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] @@ -93,19 +186,21 @@ pub struct MyFunction<'a> { impl MyFunction<'_> { fn key(&self) -> WorldKey { if let Some(interface) = self.interface.as_ref() { - WorldKey::Interface(interface.id) + // Use the stored key: inline world interfaces are keyed by name. + interface.key.clone() } else { WorldKey::Name(self.name.into()) } } } -#[derive(Copy, Clone)] +#[derive(Clone)] pub struct InterfaceInfo<'a> { package: Option>, name: &'a str, docs: Option<&'a str>, - naming: Naming, + /// Fully-qualified WIT name, e.g. `foo:bar/baz@0.1.0`. + wit_key: String, } struct FunctionCode { @@ -136,6 +231,8 @@ struct TypeLocation { pub struct Locations { types: HashMap, keys: HashMap, + /// Module owning each interface's definitions; later ones alias it. + interfaces: HashMap, } pub struct Summary<'a> { @@ -153,8 +250,8 @@ pub struct Summary<'a> { resource_info: HashMap, world_types: HashMap>, world_keys: HashMap>, - imported_interface_names: HashMap, - exported_interface_names: HashMap, + imported_interface_names: HashMap, + exported_interface_names: HashMap, imported_function_indexes: &'a HashMap<(Option<&'a str>, &'a str), usize>, exported_function_indexes: &'a HashMap<(Option<&'a str>, &'a str), usize>, stream_and_future_indexes: &'a HashMap, @@ -164,7 +261,7 @@ pub struct Summary<'a> { impl<'a> Summary<'a> { pub fn try_new( resolve: &'a Resolve, - worlds: &IndexMap, + worlds: &IndexSet, import_interface_names: &HashMap<&str, &str>, export_interface_names: &HashMap<&str, &str>, imported_function_indexes: &'a HashMap<(Option<&'a str>, &'a str), usize>, @@ -196,33 +293,140 @@ impl<'a> Summary<'a> { let mut import_keys_seen = HashSet::new(); let mut export_keys_seen = HashSet::new(); - for (&world, &naming) in worlds { + for &world in worlds { me.visit_functions( &resolve.worlds[world].imports, Direction::Import, world, &mut import_keys_seen, - naming, )?; me.visit_functions( &resolve.worlds[world].exports, Direction::Export, world, &mut export_keys_seen, - naming, )?; } me.types = me.types_sorted(); - me.imported_interface_names = me.interface_names( + let mut imports = me.interface_names( me.imported_interfaces.keys().copied(), import_interface_names, - ); - me.exported_interface_names = me.interface_names( + )?; + let mut exports = me.interface_names( me.exported_interfaces.keys().copied(), export_interface_names, - ); + )?; + + // Canonical-version ties (across both directions) fall back to full versions. + let mut flats = BTreeMap::>::new(); + for (id, named) in imports.iter().chain(&exports) { + flats + .entry(named.name.flat.clone()) + .or_default() + .insert(*id); + } + let ambiguous = flats + .into_iter() + .filter(|(_, ids)| ids.len() > 1) + .map(|(flat, _)| flat) + .collect::>(); + me.apply_version_fallback(&mut imports, &ambiguous); + me.apply_version_fallback(&mut exports, &ambiguous); + + for named in imports.values().chain(exports.values()) { + for component in &named.name.path { + if !is_python_identifier(component) || is_python_keyword(component) { + bail!( + "the module path `{}` (for `{}`) contains `{component}`, which is not \ + a valid Python identifier", + named.name.dotted(), + named.key + ); + } + } + } + + // A CamelCase override leaf would collide with its generated ABC. + for named in exports.values() { + let leaf = named.name.leaf(); + if named.overridden && leaf == leaf.to_upper_camel_case() { + bail!( + "the module name `{leaf}` (for `{}`) collides with the abstract base \ + class generated for it; use a snake_case module name", + named.key + ); + } + } + + // Report unresolved conflicts (BTreeMap keeps the choice deterministic). + let mut flats = BTreeMap::<&str, BTreeMap>::new(); + for (id, named) in imports.iter().chain(&exports) { + flats + .entry(named.name.flat.as_str()) + .or_default() + .insert(*id, named.key.as_str()); + } + if let Some((flat, ids)) = flats.iter().find(|(_, ids)| ids.len() > 1) { + bail!( + "the interfaces {} all map to the module alias `{flat}`; please specify a \ + unique name for one of them using \ + `--import-interface-name`/`--export-interface-name`", + ids.values() + .map(|key| format!("`{key}`")) + .collect::>() + .into_iter() + .collect::>() + .join(", "), + ); + } + + // Reserved: `guest` (module alias, world import) and `WorldExports` (world type). + for named in imports.values().chain(exports.values()) { + if named.name.flat == "guest" { + bail!( + "the module alias `guest` (for `{}`) collides with the reserved `guest` \ + decorator; please rename it", + named.key + ); + } + } + for &world in worlds { + let world = &resolve.worlds[world]; + for (key, item) in &world.imports { + if let (WorldItem::Function(function), WorldKey::Name(name)) = (item, key) + && function.name.to_snake_case() == "guest" + { + bail!( + "the world-level import `{name}` collides with the reserved \ + `guest` decorator; please rename it" + ); + } + } + for (key, item) in world.imports.iter().chain(&world.exports) { + if let (WorldItem::Type { .. }, WorldKey::Name(name)) = (item, key) + && name.to_upper_camel_case() == WORLD_EXPORTS_CLASS + { + bail!( + "the world-level type `{name}` collides with the reserved \ + `{WORLD_EXPORTS_CLASS}` class; please rename it" + ); + } + } + } + + Self::check_shadowing(&imports)?; + Self::check_shadowing(&exports)?; + + me.imported_interface_names = imports + .into_iter() + .map(|(id, named)| (id, named.name)) + .collect(); + me.exported_interface_names = exports + .into_iter() + .map(|(id, named)| (id, named.name)) + .collect(); Ok(me) } @@ -403,7 +607,6 @@ impl<'a> Summary<'a> { direction: Direction, world: WorldId, keys_seen: &mut HashSet, - naming: Naming, ) -> Result<()> { for (key, item) in items { self.world_keys @@ -449,7 +652,7 @@ impl<'a> Summary<'a> { package, name: item_name, docs: interface.docs.contents.as_deref(), - naming, + wit_key: self.resolve.name_world_key(key), }; self.resource_state = Some(ResourceState { direction }); @@ -503,10 +706,10 @@ impl<'a> Summary<'a> { &self, id: TypeId, ty: &TypeDef, - world_module: &str, + bindings_module: &str, reverse_cloned_interfaces: &HashMap, ) -> Option<(String, String)> { - if let Some(package) = self.package(ty.owner, world_module, reverse_cloned_interfaces) { + if let Some(package) = self.package(ty.owner, bindings_module, reverse_cloned_interfaces) { let name = if let Some(name) = &ty.name { name.to_upper_camel_case().escape() } else { @@ -522,13 +725,13 @@ impl<'a> Summary<'a> { fn summarize_resource( &self, id: TypeId, - world_module: &str, + bindings_module: &str, reverse_cloned_interfaces: &HashMap, ) -> exports::Resource { let ty = &self.resolve.types[id]; assert!(matches!(ty.kind, TypeDefKind::Resource)); let (package, name) = self - .package_and_name(id, ty, world_module, reverse_cloned_interfaces) + .package_and_name(id, ty, bindings_module, reverse_cloned_interfaces) .unwrap(); exports::Resource { package, name } @@ -537,7 +740,7 @@ impl<'a> Summary<'a> { fn summarize_record( &self, id: TypeId, - world_module: &str, + bindings_module: &str, reverse_cloned_interfaces: &HashMap, ) -> exports::Record { let ty = &self.resolve.types[id]; @@ -545,7 +748,7 @@ impl<'a> Summary<'a> { unreachable!() }; let (package, name) = self - .package_and_name(id, ty, world_module, reverse_cloned_interfaces) + .package_and_name(id, ty, bindings_module, reverse_cloned_interfaces) .unwrap(); exports::Record { @@ -562,7 +765,7 @@ impl<'a> Summary<'a> { fn summarize_flags( &self, id: TypeId, - world_module: &str, + bindings_module: &str, reverse_cloned_interfaces: &HashMap, ) -> exports::Flags { let ty = &self.resolve.types[id]; @@ -570,7 +773,7 @@ impl<'a> Summary<'a> { unreachable!() }; let (package, name) = self - .package_and_name(id, ty, world_module, reverse_cloned_interfaces) + .package_and_name(id, ty, bindings_module, reverse_cloned_interfaces) .unwrap(); exports::Flags { @@ -583,7 +786,7 @@ impl<'a> Summary<'a> { fn summarize_variant( &self, id: TypeId, - world_module: &str, + bindings_module: &str, reverse_cloned_interfaces: &HashMap, ) -> exports::Variant { let ty = &self.resolve.types[id]; @@ -591,7 +794,7 @@ impl<'a> Summary<'a> { unreachable!() }; let (package, name) = self - .package_and_name(id, ty, world_module, reverse_cloned_interfaces) + .package_and_name(id, ty, bindings_module, reverse_cloned_interfaces) .unwrap(); let cases = variant @@ -613,7 +816,7 @@ impl<'a> Summary<'a> { fn summarize_enum( &self, id: TypeId, - world_module: &str, + bindings_module: &str, reverse_cloned_interfaces: &HashMap, ) -> exports::Enum { let ty = &self.resolve.types[id]; @@ -621,7 +824,7 @@ impl<'a> Summary<'a> { unreachable!() }; let (package, name) = self - .package_and_name(id, ty, world_module, reverse_cloned_interfaces) + .package_and_name(id, ty, bindings_module, reverse_cloned_interfaces) .unwrap(); exports::Enum { @@ -673,10 +876,11 @@ impl<'a> Summary<'a> { let mut map = HashMap::new(); for function in &self.functions { if let FunctionKind::Export = function.kind { + // Registry key the app's `guest` decorator registered under. let scope = if let Some(interface) = &function.interface { - &self.exported_interface_names[&interface.id] + self.resolve.name_world_key(interface.key) } else { - locations.keys.get(&function.key()).unwrap() + locations.keys.get(&function.key()).unwrap().clone() }; map.insert( @@ -692,19 +896,16 @@ impl<'a> Summary<'a> { wit_parser::FunctionKind::Freestanding | wit_parser::FunctionKind::AsyncFreestanding => { FunctionExportKind::Freestanding(Function { - protocol: scope.to_upper_camel_case().escape(), + scope, name: self.function_name_for_call(function), }) } wit_parser::FunctionKind::Constructor(id) => { FunctionExportKind::Constructor(Constructor { - module: scope.to_snake_case().escape(), - protocol: self.resolve.types[id] - .name - .as_deref() - .unwrap() - .to_upper_camel_case() - .escape(), + scope: resource_scope( + &scope, + self.resolve.types[id].name.as_deref().unwrap(), + ), }) } wit_parser::FunctionKind::Method(_) @@ -714,13 +915,10 @@ impl<'a> Summary<'a> { wit_parser::FunctionKind::Static(id) | wit_parser::FunctionKind::AsyncStatic(id) => { FunctionExportKind::Static(Static { - module: scope.to_snake_case().escape(), - protocol: self.resolve.types[id] - .name - .as_deref() - .unwrap() - .to_upper_camel_case() - .escape(), + scope: resource_scope( + &scope, + self.resolve.types[id].name.as_deref().unwrap(), + ), name: self.function_name_for_call(function), }) } @@ -931,8 +1129,7 @@ impl<'a> Summary<'a> { fn function_code( &self, - direction: Direction, - world_module: &str, + bindings_module: &str, function: &MyFunction, names: &mut TypeNames, seen: &HashSet, @@ -971,24 +1168,24 @@ impl<'a> Summary<'a> { let mut type_name = |ty| names.type_name(ty, seen, resource); + // Absolute, so unlike a flat alias it must name this copy's own tree. + let direction = if let FunctionKind::Export = function.kind { + Direction::Export + } else { + Direction::Import + }; + let absolute_type_name = |ty| { - format!( - "{world_module}.{}.{}", - match direction { - Direction::Import => "imports", - Direction::Export => "exports", - }, - TypeNames::new(self, TypeOwner::None).type_name( - ty, - &if let Type::Id(id) = ty { - Some(dealias(self.resolve, id)) - } else { - None - } - .into_iter() - .collect::>(), + TypeNames::new_absolute(self, bindings_module, direction).type_name( + ty, + &if let Type::Id(id) = ty { + Some(dealias(self.resolve, id)) + } else { None - ) + } + .into_iter() + .collect::>(), + None, ) }; @@ -1194,12 +1391,13 @@ impl<'a> Summary<'a> { sorted } + /// Fully qualified paths; a canonical-version tie is the one thing that can rename them. fn interface_names( &self, ids: impl Iterator, interface_names: &HashMap<&str, &str>, - ) -> HashMap { - let mut tree = HashMap::<_, HashMap<_, HashMap<_, _>>>::new(); + ) -> Result> { + let mut names = HashMap::new(); for id in ids { let info = if let Some(info) = self.imported_interfaces.get(&id) { info @@ -1209,74 +1407,118 @@ impl<'a> Summary<'a> { unreachable!() }; - assert!( - tree.entry(info.name) - .or_default() - .entry(info.package.map(|p| (p.namespace, p.name))) - .or_default() - .insert(info.package.and_then(|p| p.version), (id, info.naming)) - .is_none() - ); - } + let named = if let Some(package) = info.package { + let PackageName { + namespace, + name, + version, + } = package; + let interface = info.name; - let mut names = HashMap::new(); - for (name, packages) in &tree { - for (package, versions) in packages { - if let Some((package_namespace, package_name)) = package { - for (version, &(id, naming)) in versions { - assert!( - names - .insert( - id, - if let Some(version) = version { - if let Some(name) = interface_names.get( - format!( - "{package_namespace}:{package_name}\ - /{name}@{version}" - ) - .as_str(), - ) { - (*name).to_owned() - } else if versions.len() == 1 && naming == Naming::Short { - if packages.len() == 1 { - (*name).to_owned() - } else { - format!("{package_namespace}-{package_name}-{name}") - } - } else { - format!( - "{package_namespace}-{package_name}-{name}-{}", - version.to_string().replace('.', "-") - ) - } - } else if let Some(name) = interface_names.get( - format!("{package_namespace}:{package_name}/{name}") - .as_str() - ) { - (*name).to_owned() - } else if packages.len() == 1 && naming == Naming::Short { - (*name).to_owned() - } else { - format!("{package_namespace}-{package_name}-{name}",) - } - ) - .is_none() - ); + let key = if let Some(version) = version { + format!("{namespace}:{name}/{interface}@{version}") + } else { + format!("{namespace}:{name}/{interface}") + }; + + if let Some(name) = interface_names.get(key.as_str()) { + NamedInterface { + name: InterfaceName::from_override(name), + key, + overridden: true, } } else { - assert!( - names - .insert( - versions.get(&None).unwrap().0, - (*interface_names.get(*name).unwrap_or(name)).to_owned() - ) - .is_none() + // Version goes on the package label: `from wit.imports.foo import bar_v0_3`. + let package = if let Some(version) = version { + format!("{name}-{}", canonical_version(version)) + } else { + (*name).to_owned() + }; + + NamedInterface { + name: InterfaceName::new([ + (*namespace).to_owned(), + package, + interface.to_owned(), + ]), + key, + overridden: false, + } + } + } else { + // Unqualified interface (not in any package): top level. + let key = info.name.to_owned(); + if let Some(name) = interface_names.get(info.name) { + NamedInterface { + name: InterfaceName::from_override(name), + key, + overridden: true, + } + } else { + NamedInterface { + name: InterfaceName::new([info.name.to_owned()]), + key, + overridden: false, + } + } + }; + + assert!(names.insert(id, named).is_none()); + } + + Ok(names) + } + + /// Rename eligible members of canonical-version ties to their full versions. + fn apply_version_fallback( + &self, + names: &mut HashMap, + ambiguous: &HashSet, + ) { + for (id, named) in names { + if ambiguous.contains(&named.name.flat) && !named.overridden { + let info = self + .imported_interfaces + .get(id) + .unwrap_or_else(|| &self.exported_interfaces[id]); + if let Some(package) = info.package + && let Some(version) = package.version + { + named.name = InterfaceName::new([ + package.namespace.to_owned(), + format!("{}-{}", package.name, full_version(version)), + info.name.to_owned(), + ]); + } + } + } + } + + /// A path prefixing another would be both module and package; the module loses. + fn check_shadowing(names: &HashMap) -> Result<()> { + let paths = names + .values() + .map(|named| (named.name.dotted(), named)) + .collect::>(); + let mut sorted = names.values().collect::>(); + sorted.sort_by_key(|named| named.name.dotted()); + for named in sorted { + for count in 1..named.name.path.len() { + if let Some(other) = paths.get(&named.name.path[..count].join(".")) { + bail!( + "the module `{}` (for `{}`) would be shadowed by the package containing \ + `{}` (for `{}`); please specify a different name for one of them using \ + `--import-interface-name`/`--export-interface-name`", + other.name.dotted(), + other.key, + named.name.dotted(), + named.key, ); } } } - names + Ok(()) } #[allow(clippy::too_many_arguments)] @@ -1403,10 +1645,21 @@ impl<'a> Summary<'a> { &self, path: &Path, world: WorldId, - world_module: &str, + bindings_module: &str, locations: &mut Locations, stub_runtime_calls: bool, ) -> Result<()> { + // Exported interfaces must not clash with the world-level registry key. + for info in self.exported_interfaces.values() { + if info.wit_key == bindings_module { + bail!( + "the exported interface `{bindings_module}` collides with the bindings \ + module name, which world-level exports register under; please rename \ + one of them (e.g. via `--bindings-module`)" + ); + } + } + #[derive(Default)] struct Definitions<'a> { types: Vec, @@ -1417,10 +1670,6 @@ impl<'a> Summary<'a> { alias_module: Option, } - let file_header = "# This file is automatically generated by componentize-py -# It is not intended for manual editing. -"; - let mut interface_imports = HashMap::::new(); let mut interface_exports = HashMap::::new(); let mut world_imports = Definitions::default(); @@ -1618,8 +1867,7 @@ class {camel}(Flag): class_method, error, } = self.function_code( - Direction::Import, - world_module, + bindings_module, function, &mut names, &seen, @@ -1722,8 +1970,7 @@ class {camel}: error, .. } = self.function_code( - Direction::Export, - world_module, + bindings_module, function, &mut names, &seen, @@ -1875,7 +2122,7 @@ def {snake}_future(default: Callable[[], {camel}]) -> tuple[FutureWriter[{camel} }; let aliases = if let (Some(code), false) = (code.as_ref(), names.is_empty()) { - let aliases = iter::once(world_module_import(world_module, "peer")) + let aliases = iter::once(bindings_module_import(bindings_module, "peer")) .chain( names .iter() @@ -1900,7 +2147,7 @@ def {snake}_future(default: Callable[[], {camel}]) -> tuple[FutureWriter[{camel} locations.types.insert( id, TypeLocation { - module: world_module.to_owned(), + module: bindings_module.to_owned(), aliases, }, ); @@ -2005,14 +2252,7 @@ def {snake}_future(default: Callable[[], {camel}]) -> tuple[FutureWriter[{camel} return_statement, error, .. - } = self.function_code( - Direction::Import, - world_module, - function, - &mut names, - &seen, - None, - ); + } = self.function_code(bindings_module, function, &mut names, &seen, None); match function.kind { FunctionKind::Import => { @@ -2063,9 +2303,9 @@ def {snake}_future(default: Callable[[], {camel}]) -> tuple[FutureWriter[{camel} let module = locations .keys .entry(key) - .or_insert_with(|| world_module.to_owned()); + .or_insert_with(|| bindings_module.to_owned()); - if module == world_module { + if module == bindings_module { let params = if params.is_empty() { "self".to_owned() } else { @@ -2098,7 +2338,7 @@ def {snake}_future(default: Callable[[], {camel}]) -> tuple[FutureWriter[{camel} } let python_imports = - "from typing import TypeVar, Generic, Union, Optional, Protocol, Tuple, List, Mapping, Any, Self, Callable + "from typing import TypeVar, Generic, Union, Optional, Protocol, Tuple, Mapping, List, Any, Self, Callable, ClassVar from types import TracebackType from enum import Flag, Enum, auto from dataclasses import dataclass @@ -2110,19 +2350,35 @@ import weakref from componentize_py_async_support.streams import StreamReader, StreamWriter, ByteStreamReader, ByteStreamWriter from componentize_py_async_support.futures import FutureReader, FutureWriter"; - let import = |prefix, interface| { - let (module, package) = self.interface_package(interface); - format!("from {prefix}{module} import {package}") + // Aliases bind import-first: shared two-way types exist only in the imports tree. + let import = |interface| { + let (module, name) = self.interface_name(interface); + let leaf = name.leaf(); + let alias = if leaf == name.flat { + String::new() + } else { + format!(" as {}", name.flat) + }; + if let Some(parent) = name.parent() { + format!("from {bindings_module}.{module}.{parent} import {leaf}{alias}") + } else { + format!("from {bindings_module}.{module} import {leaf}{alias}") + } }; if !interface_imports.is_empty() { let dir = path.join("imports"); - fs::create_dir(&dir)?; + fs::create_dir_all(&dir)?; File::create(dir.join("__init__.py"))?; + let mut interface_ids = Vec::new(); for (id, code) in interface_imports.into_iter().collect::>() { + interface_ids.push(id); + locations + .interfaces + .entry(id) + .or_insert_with(|| bindings_module.to_owned()); let name = self.imported_interface_names.get(&id).unwrap(); - let mut file = - File::create(dir.join(format!("{}.py", name.to_snake_case().escape())))?; + let mut file = File::create(create_package(&dir, name)?)?; let types = code.types.concat(); let functions = code.functions.concat(); let imports = code @@ -2130,7 +2386,7 @@ from componentize_py_async_support.futures import FutureReader, FutureWriter"; .union(&code.function_imports) .collect::>() .into_iter() - .map(|&interface| import("..", interface)) + .map(|&interface| import(interface)) .chain(self.need_async.then(|| async_imports.into())) .chain((!stub_runtime_calls).then(|| "import componentize_py_runtime".into())) .collect::>() @@ -2139,7 +2395,7 @@ from componentize_py_async_support.futures import FutureReader, FutureWriter"; write!( file, - "{file_header}{docs}{python_imports} + "{FILE_HEADER}{docs}{python_imports} from componentize_py_types import Result, Ok, Err, Some {imports} {types} @@ -2147,25 +2403,51 @@ from componentize_py_types import Result, Ok, Err, Some " )?; } + + bind_submodules( + &dir, + interface_ids + .iter() + .map(|id| &self.imported_interface_names[id]), + )?; } if !interface_exports.is_empty() { let dir = path.join("exports"); - fs::create_dir(&dir)?; + fs::create_dir_all(&dir)?; + File::create(dir.join("__init__.py"))?; - let mut protocol_imports = HashSet::new(); - let mut protocols = String::new(); + // Each package's `__init__.py` holds the ABCs of its interfaces. + let mut protocols = BTreeMap::, (HashSet<_>, String)>::new(); + let mut interface_ids = Vec::new(); for (id, code) in interface_exports.into_iter().collect::>() { + interface_ids.push(id); + locations + .interfaces + .entry(id) + .or_insert_with(|| bindings_module.to_owned()); let name = self.exported_interface_names.get(&id).unwrap(); - let mut file = - File::create(dir.join(format!("{}.py", name.to_snake_case().escape())))?; + let guest = if let Some(alias_module) = &code.alias_module { + let peer = if let Some(parent) = name.parent() { + format!("{alias_module}.exports.{parent}") + } else { + format!("{alias_module}.exports") + }; + format!( + "\nfrom {peer} import {} as _peer\nguest = _peer.guest\n", + name.leaf() + ) + } else { + self.guest_code(id) + }; + let mut file = File::create(create_package(&dir, name)?)?; let types = code.types.concat(); let imports = code .type_imports .into_iter() .collect::>() .into_iter() - .map(|interface| import("..", interface)) + .map(&import) .chain(self.need_async.then(|| async_imports.into())) .collect::>() .join("\n"); @@ -2173,28 +2455,45 @@ from componentize_py_types import Result, Ok, Err, Some write!( file, - "{file_header}{docs}{python_imports} + "{FILE_HEADER}{docs}{python_imports} from componentize_py_types import Result, Ok, Err, Some +import componentize_py_exports {imports} {types} -" +{guest}" )?; - let camel = name.to_upper_camel_case().escape(); + let camel = name.leaf().to_upper_camel_case().escape(); + let (protocol_imports, protocols) = protocols.entry(name.parent()).or_default(); if let Some(alias_module) = code.alias_module { - writeln!( - &mut protocols, - "import {}", - if let Some((start, _)) = alias_module.split_once('.') { - start - } else { - &alias_module - } - )?; - writeln!(&mut protocols, "{camel} = {alias_module}.{camel}")?; + // Exported by a peer module; alias its ABC. + let peer = if let Some(parent) = name.parent() { + format!("{alias_module}.exports.{parent}") + } else { + format!("{alias_module}.exports") + }; + writeln!(protocols, "import {peer}")?; + writeln!(protocols, "{camel} = {peer}.{camel}")?; } else { - let methods = if code.functions.is_empty() { + // Import each resource ABC directly: flat aliases resolve import-first. + let mut resources = String::new(); + for resource in self.exported_resources(id) { + let resource_camel = resource.to_upper_camel_case().escape(); + let alias = format!("_{}_{}", name.flat, resource.to_snake_case().escape()); + writeln!( + protocols, + "from .{} import {resource_camel} as {alias}", + name.leaf() + )?; + writeln!( + &mut resources, + " {}: ClassVar[type[{alias}]]", + resource.to_snake_case().escape() + )?; + } + + let methods = if code.functions.is_empty() && resources.is_empty() { " pass".to_owned() } else { code.functions.concat() @@ -2202,32 +2501,45 @@ from componentize_py_types import Result, Ok, Err, Some protocol_imports.extend(code.function_imports); write!( - &mut protocols, + protocols, " class {camel}(Protocol): -{methods} +{resources}{methods} " )?; } } - let mut init = File::create(dir.join("__init__.py"))?; - let imports = protocol_imports - .into_iter() - .collect::>() - .into_iter() - .map(|interface| import("..", interface)) - .chain(self.need_async.then(|| async_imports.into())) - .collect::>() - .join("\n"); + for (package, (protocol_imports, protocols)) in protocols { + let dir = if let Some(package) = &package { + dir.join(package.replace('.', "/")) + } else { + dir.clone() + }; + let mut init = File::create(dir.join("__init__.py"))?; + let imports = protocol_imports + .into_iter() + .collect::>() + .into_iter() + .map(&import) + .chain(self.need_async.then(|| async_imports.into())) + .collect::>() + .join("\n"); - write!( - init, - "{file_header}{python_imports} + write!( + init, + "{FILE_HEADER}{python_imports} from componentize_py_types import Result, Ok, Err, Some {imports} {protocols} " + )?; + } + bind_submodules( + &dir, + interface_ids + .iter() + .map(|id| &self.exported_interface_names[id]), )?; } @@ -2235,10 +2547,14 @@ from componentize_py_types import Result, Ok, Err, Some let mut file = File::create(path.join("__init__.py"))?; let function_imports = world_imports.functions.concat(); let type_exports = world_exports.types.concat(); - let camel = world_module.to_upper_camel_case().escape(); + let camel = WORLD_EXPORTS_CLASS; - let protocol = if let Some(alias_module) = world_exports.alias_module { - format!("{camel} = {alias_module}.{camel}") + let protocol = if let Some(alias_module) = &world_exports.alias_module { + // World exports owned by a peer; alias its decorator (peer's key). + format!( + "import {alias_module}\n{camel} = {alias_module}.{camel}\n\ + guest = {alias_module}.guest" + ) } else { let methods = if world_exports.functions.is_empty() { " pass".to_owned() @@ -2247,34 +2563,52 @@ from componentize_py_types import Result, Ok, Err, Some }; format!( - "class {camel}(Protocol): -{methods}" + r#"class {camel}(Protocol): +{methods} + +_GuestT = TypeVar("_GuestT") + +class _Guest: + """Registers a class as the implementation of this world's top-level exports.""" + + def __call__(self, value: type[_GuestT]) -> type[_GuestT]: + return componentize_py_exports.register({bindings_module:?})(value) + +guest = _Guest()"# ) }; - let imports = world_imports - .function_imports - .union( - &world_exports - .type_imports - .union(&world_exports.function_imports) - .copied() - .collect(), - ) - .collect::>() - .into_iter() - .map(|&interface| import(".", interface)) - .chain(self.need_async.then(|| async_imports.into())) - .chain((!stub_runtime_calls).then(|| "import componentize_py_runtime".into())) - .collect::>() - .join("\n"); + let imports = [ + &world_imports.type_imports, + &world_imports.function_imports, + &world_exports.type_imports, + &world_exports.function_imports, + ] + .into_iter() + .flatten() + .collect::>() + .into_iter() + .map(|&interface| import(interface)) + .chain(self.need_async.then(|| async_imports.into())) + .chain((!stub_runtime_calls).then(|| "import componentize_py_runtime".into())) + .collect::>() + .join("\n"); let docs = docstring(world_exports.docs, 0, None); + let subpackages = ["imports", "exports"] + .iter() + .filter(|name| path.join(name).is_dir()) + .map(|name| format!("from . import {name}")) + .collect::>() + .join("\n"); + write!( file, - "{file_header}{docs}{python_imports} + "{FILE_HEADER}{docs}{python_imports} from componentize_py_types import Result, Ok, Err, Some +import componentize_py_exports +{subpackages} {imports} {type_exports} {function_imports} @@ -2286,23 +2620,122 @@ from componentize_py_types import Result, Ok, Err, Some Ok(()) } - fn interface_package(&self, interface: InterfaceId) -> (&'static str, String) { + fn exported_resources(&self, interface: InterfaceId) -> Vec<&str> { + let empty = &ResourceInfo::default(); + self.resolve.interfaces[interface] + .types + .values() + .filter(|&&id| { + matches!(self.resolve.types[id].kind, TypeDefKind::Resource) + && self.resource_info.get(&id).unwrap_or(empty).local + }) + .map(|&id| self.resolve.types[id].name.as_deref().unwrap()) + .collect() + } + + /// Emit the reserved `guest` decorator appended to an exported interface's module. + fn guest_code(&self, interface: InterfaceId) -> String { + let key = &self.exported_interfaces[&interface].wit_key; + + let resources = self + .exported_resources(interface) + .into_iter() + .map(|resource| { + format!( + " ({:?}, {:?}, {:?}),\n", + resource.to_snake_case().escape(), + resource, + resource_scope(key, resource) + ) + }) + .collect::>() + .concat(); + + format!( + r#" +_GUEST_RESOURCES: Tuple[Tuple[str, str, str], ...] = ( +{resources}) + +_GuestT = TypeVar("_GuestT") + +class _Guest: + """Registers a class as the implementation of `{key}`.""" + + def __call__(self, value: type[_GuestT]) -> type[_GuestT]: + componentize_py_exports.register({key:?})(value) + for attr, resource, scope in _GUEST_RESOURCES: + impl = next( + (cls.__dict__[attr] for cls in value.__mro__ if attr in cls.__dict__), None + ) + if impl is None: + raise RuntimeError( + f"`{{value.__name__}}` is registered as the implementation of " + f"`{key}`, so it must declare a class attribute `{{attr}}` " + f"naming the class which implements the `{{resource}}` resource" + ) + componentize_py_exports.register(scope)(impl) + return value + +guest = _Guest() +"# + ) + } + + /// Placeholder and module path for each imported `HELPER_INTERFACES` entry, + /// resolved to the module owning it so the answer is unique. + pub fn helper_module_paths(&self, locations: &Locations) -> Vec<(&'static str, String)> { + HELPER_INTERFACES + .iter() + .filter_map(|(placeholder, wanted)| { + let (id, _) = self.imported_interfaces.iter().find(|(_, info)| { + info.name == wanted.interface + && info.package.is_some_and(|package| { + package.namespace == "wasi" + && package.name == wanted.package + // The helpers only support 0.2.x; see poll_loop.py. + && package + .version + .is_some_and(|v| v.major == 0 && v.minor == 2) + }) + })?; + + let module = locations.interfaces.get(id)?; + let name = self.imported_interface_names.get(id)?; + Some((*placeholder, format!("{module}.imports.{}", name.dotted()))) + }) + .collect() + } + + /// Like `interface_name`, but uses `direction`'s tree where `ty` is + /// duplicated into it; anything else lives in exactly one tree. + fn interface_name_in( + &self, + direction: Direction, + interface: InterfaceId, + ty: Type, + ) -> (&'static str, &InterfaceName) { + if let Direction::Export = direction + && self.has_imported_and_exported_resource(ty) + && let Some(name) = self.exported_interface_names.get(&interface) + { + ("exports", name) + } else { + self.interface_name(interface) + } + } + + fn interface_name(&self, interface: InterfaceId) -> (&'static str, &InterfaceName) { if let Some(name) = self.imported_interface_names.get(&interface) { - ("imports", name.to_snake_case().escape()) + ("imports", name) } else { - ( - "exports", - self.exported_interface_names[&interface] - .to_snake_case() - .escape(), - ) + ("exports", &self.exported_interface_names[&interface]) } } fn package( &self, owner: TypeOwner, - world_module: &str, + bindings_module: &str, reverse_cloned_interfaces: &HashMap, ) -> Option { match owner { @@ -2310,10 +2743,10 @@ from componentize_py_types import Result, Ok, Err, Some if let Some(&original) = reverse_cloned_interfaces.get(&interface) { interface = original; } - let (module, package) = self.interface_package(interface); - Some(format!("{world_module}.{module}.{package}")) + let (module, name) = self.interface_name(interface); + Some(format!("{bindings_module}.{module}.{}", name.dotted())) } - TypeOwner::World(_) => Some(world_module.to_owned()), + TypeOwner::World(_) => Some(bindings_module.to_owned()), TypeOwner::None => None, } } @@ -2387,6 +2820,10 @@ struct TypeNames<'a> { summary: &'a Summary<'a>, owner: TypeOwner, imports: HashSet, + /// Root absolute type paths here instead of flat aliases (for docstrings). + absolute: Option<&'a str>, + /// Tree of the code being named; only consulted for absolute paths. + direction: Direction, } impl<'a> TypeNames<'a> { @@ -2395,6 +2832,21 @@ impl<'a> TypeNames<'a> { summary, owner, imports: HashSet::new(), + absolute: None, + direction: Direction::Import, + } + } + + /// Absolute paths rooted at `bindings_module`, resolved in `direction`. + fn new_absolute( + summary: &'a Summary<'_>, + bindings_module: &'a str, + direction: Direction, + ) -> Self { + Self { + absolute: Some(bindings_module), + direction, + ..Self::new(summary, TypeOwner::None) } } @@ -2427,7 +2879,19 @@ impl<'a> TypeNames<'a> { match ty.owner { TypeOwner::Interface(interface) => { self.imports.insert(interface); - format!("{}.", self.summary.interface_package(interface).1) + if let Some(bindings_module) = self.absolute { + let (module, name) = self.summary.interface_name_in( + self.direction, + interface, + Type::Id(id), + ); + format!("{bindings_module}.{module}.{}.", name.dotted()) + } else { + format!( + "{}.", + self.summary.interface_name(interface).1.flat + ) + } } // todo: place anonymous types in types.py // and import them from there @@ -2554,7 +3018,7 @@ impl<'a> TypeNames<'a> { } else { match ty.owner { TypeOwner::Interface(interface) => { - format!("{}_", self.summary.interface_package(interface).1) + format!("{}_", self.summary.interface_name(interface).1.flat) } _ => String::new(), } @@ -2626,18 +3090,54 @@ pub trait Escape { fn escape(self) -> Self; } +// Source: https://docs.python.org/3/reference/lexical_analysis.html#keywords +pub fn is_python_keyword(s: &str) -> bool { + matches!( + s, + "False" + | "None" + | "True" + | "and" + | "as" + | "assert" + | "async" + | "await" + | "break" + | "class" + | "continue" + | "def" + | "del" + | "elif" + | "else" + | "except" + | "finally" + | "for" + | "from" + | "global" + | "if" + | "import" + | "in" + | "is" + | "lambda" + | "nonlocal" + | "not" + | "or" + | "pass" + | "raise" + | "return" + | "try" + | "while" + | "with" + | "yield" + ) +} + impl Escape for String { fn escape(self) -> Self { - // Escape Python keywords; source: - // https://docs.python.org/3/reference/lexical_analysis.html#keywords - match self.as_str() { - "False" | "None" | "True" | "and" | "as" | "assert" | "async" | "await" | "break" - | "class" | "continue" | "def" | "del" | "elif" | "else" | "except" | "finally" - | "for" | "from" | "global" | "if" | "import" | "in" | "is" | "lambda" | "nonlocal" - | "not" | "or" | "pass" | "raise" | "return" | "try" | "while" | "with" | "yield" => { - format!("{self}_") - } - _ => self, + if is_python_keyword(&self) { + format!("{self}_") + } else { + self } } } @@ -2659,7 +3159,64 @@ fn matches_resource(function: &MyFunction, resource: TypeId, direction: Directio } } -fn world_module_import(name: &str, alias: &str) -> String { +/// Semver-significant part (`0.1.7` -> `v0-1`); full version if prerelease/build. +fn canonical_version(version: &Version) -> String { + if !(version.pre.is_empty() && version.build.is_empty()) { + full_version(version) + } else if version.major > 0 { + format!("v{}", version.major) + } else if version.minor > 0 { + format!("v0-{}", version.minor) + } else { + format!("v0-0-{}", version.patch) + } +} + +/// Full version with non-alphanumerics replaced by `-`. +fn full_version(version: &Version) -> String { + format!("v{version}") + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect() +} + +/// Create package dirs and missing `__init__.py`s; returns the module file path. +/// Append `from . import ` lines so packages expose their submodules. +fn bind_submodules<'a>(dir: &Path, names: impl Iterator) -> Result<()> { + let mut children = BTreeMap::>::new(); + for name in names { + let mut dir = dir.to_owned(); + for component in &name.path { + children.entry(dir.clone()).or_default().insert(component); + dir = dir.join(component); + } + } + + for (dir, children) in children { + let mut init = fs::OpenOptions::new() + .append(true) + .open(dir.join("__init__.py"))?; + for child in children { + writeln!(init, "from . import {child}")?; + } + } + + Ok(()) +} + +fn create_package(dir: &Path, name: &InterfaceName) -> Result { + let mut dir = dir.to_owned(); + for component in &name.path[..name.path.len() - 1] { + dir = dir.join(component); + fs::create_dir_all(&dir)?; + // Truncate: these are generated-owned, and reruns must not accumulate. + File::create(dir.join("__init__.py"))?; + } + + Ok(dir.join(format!("{}.py", name.leaf()))) +} + +fn bindings_module_import(name: &str, alias: &str) -> String { if let Some((front, rear)) = name.rsplit_once('.') { format!("from {front} import {rear} as {alias}") } else { @@ -2736,3 +3293,45 @@ fn is_option(resolve: &Resolve, ty: Type) -> bool { false } } + +#[cfg(test)] +mod tests { + use super::{canonical_version, full_version}; + use {anyhow::Result, heck::ToSnakeCase, semver::Version}; + + #[test] + fn canonical_versions() -> Result<()> { + assert_eq!("v1", canonical_version(&Version::parse("1.0.0")?)); + assert_eq!("v1", canonical_version(&Version::parse("1.2.3")?)); + assert_eq!("v0-1", canonical_version(&Version::parse("0.1.0")?)); + assert_eq!("v0-1", canonical_version(&Version::parse("0.1.7")?)); + assert_eq!("v0-0-3", canonical_version(&Version::parse("0.0.3")?)); + + assert_eq!( + "v1-0-0-alpha-1", + canonical_version(&Version::parse("1.0.0-alpha.1")?) + ); + assert_eq!( + "v1-0-0-build-5", + canonical_version(&Version::parse("1.0.0+build.5")?) + ); + + Ok(()) + } + + #[test] + fn versions_are_valid_module_name_components() -> Result<()> { + for version in ["1.2.3", "0.1.7", "0.0.3", "1.0.0-alpha.1", "1.0.0+build.5"] { + let version = Version::parse(version)?; + for name in [canonical_version(&version), full_version(&version)] { + let name = format!("foo-bar-{name}-baz").to_snake_case(); + assert!( + name.chars().all(|c| c == '_' || c.is_ascii_alphanumeric()), + "{name} is not a valid Python identifier" + ); + } + } + + Ok(()) + } +} diff --git a/wit/init.wit b/wit/init.wit index 2f5de1ae..7cc2f34f 100644 --- a/wit/init.wit +++ b/wit/init.wit @@ -4,19 +4,19 @@ world init { import wasi:cli/environment@0.2.0; export exports: interface { + // `scope`: registry key -- interface name, bindings module for + // world-level exports, or `#` for a resource. record function { - protocol: string, + scope: string, name: string } record %constructor { - module: string, - protocol: string + scope: string } record %static { - module: string, - protocol: string, + scope: string, name: string } From 93bcc7ad26043873200f0fe7df4309dab338671a Mon Sep 17 00:00:00 2001 From: Mendy Berger <12537668+MendyBerger@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:40:08 -0400 Subject: [PATCH 2/4] Codegen update to #218 (updated tests) --- README.md | 9 ++- examples/cli-p3/app.py | 5 +- examples/cli/app.py | 5 +- examples/http-p3/app.py | 17 +++-- examples/http/app.py | 9 ++- examples/matrix-math/app.py | 12 +-- examples/sandbox/guest.py | 5 +- examples/tcp-p3/app.py | 11 +-- examples/tcp/app.py | 5 +- src/command.rs | 10 +-- src/test.rs | 9 +-- src/test/bar_sdk/componentize-py.toml | 1 - src/test/echoes.rs | 6 +- src/test/python_source/app.py | 75 +++++++++++++------ src/test/python_source/resource_aggregates.py | 4 +- src/test/python_source/resource_alias1.py | 4 +- .../python_source/resource_borrow_export.py | 2 +- .../resource_borrow_in_record.py | 4 +- .../python_source/resource_floats_exports.py | 6 +- .../resource_import_and_export.py | 4 +- src/test/python_source/resource_with_lists.py | 4 +- src/test/python_source/streams_and_futures.py | 2 +- src/test/tests.rs | 10 ++- test-generator/src/lib.rs | 6 +- tests/bindings.rs | 18 ++--- 25 files changed, 146 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index 630f9984..6f4e4a61 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,15 @@ world, you can generate them using the `bindings` subcommand: componentize-py -d hello.wit -w hello bindings hello_guest ``` -Then, use the `hello` module produced by the command above to write your app: +Then, use the bindings produced by the command above (a `wit` package inside +`hello_guest`) to write your app: ```shell cat >app.py < str: return "Hello, World!" EOF diff --git a/examples/cli-p3/app.py b/examples/cli-p3/app.py index caf22c75..77a8ba52 100644 --- a/examples/cli-p3/app.py +++ b/examples/cli-p3/app.py @@ -1,5 +1,6 @@ -from wit_world import exports +from wit.exports.wasi.cli_v0_3 import run, Run -class Run(exports.Run): +@run.guest +class Cli(Run): async def run(self) -> None: print("Hello, world!") diff --git a/examples/cli/app.py b/examples/cli/app.py index fdbed9bd..8e1b0bb7 100644 --- a/examples/cli/app.py +++ b/examples/cli/app.py @@ -1,5 +1,6 @@ -from wit_world import exports +from wit.exports.wasi.cli_v0_2 import run, Run -class Run(exports.Run): +@run.guest +class Cli(Run): def run(self) -> None: print("Hello, world!") diff --git a/examples/http-p3/app.py b/examples/http-p3/app.py index d65fdc05..6881a1b2 100644 --- a/examples/http-p3/app.py +++ b/examples/http-p3/app.py @@ -8,15 +8,15 @@ import asyncio import hashlib import componentize_py_async_support -import wit_world +import wit from typing import Optional from componentize_py_types import Ok, Result from componentize_py_async_support.streams import ByteStreamWriter from componentize_py_async_support.futures import FutureReader -from wit_world import exports -from wit_world.imports import client -from wit_world.imports.wasi_http_types import ( +from wit.exports.wasi.http_v0_3 import handler, Handler +from wit.imports.wasi.http_v0_3 import client +from wit.imports.wasi.http_v0_3.types import ( Method_Get, Method_Post, Scheme, @@ -31,7 +31,8 @@ from urllib import parse -class Handler(exports.Handler): +@handler.guest +class HttpHandler(Handler): """Implements the `export`ed portion of the `wasi-http` `proxy` world.""" async def handle(self, request: Request) -> Response: @@ -52,7 +53,7 @@ async def handle(self, request: Request) -> Response: filter(lambda pair: pair[0] == "url", headers), )) - tx, rx = wit_world.byte_stream() + tx, rx = wit.byte_stream() componentize_py_async_support.spawn(hash_all(urls, tx)) return Response.new( @@ -127,8 +128,8 @@ async def sha256(url: str) -> tuple[str, str]: def trailers_future() -> FutureReader[Result[Optional[Fields], ErrorCode]]: - return wit_world.result_option_wasi_http_types_fields_wasi_http_types_error_code_future(lambda: Ok(None))[1] + return wit.result_option_wasi_http_v0_3_types_fields_wasi_http_v0_3_types_error_code_future(lambda: Ok(None))[1] def unit_future() -> FutureReader[Result[None, ErrorCode]]: - return wit_world.result_unit_wasi_http_types_error_code_future(lambda: Ok(None))[1] + return wit.result_unit_wasi_http_v0_3_types_error_code_future(lambda: Ok(None))[1] diff --git a/examples/http/app.py b/examples/http/app.py index be720f38..7225d30b 100644 --- a/examples/http/app.py +++ b/examples/http/app.py @@ -10,9 +10,9 @@ import poll_loop from componentize_py_types import Ok -from wit_world import exports -from wit_world.imports import types -from wit_world.imports.types import ( +from wit.exports.wasi.http_v0_2 import incoming_handler, IncomingHandler +from wit.imports.wasi.http_v0_2 import types +from wit.imports.wasi.http_v0_2.types import ( Method_Get, Method_Post, Scheme, @@ -31,7 +31,8 @@ from urllib import parse -class IncomingHandler(exports.IncomingHandler): +@incoming_handler.guest +class Handler(IncomingHandler): """Implements the `export`ed portion of the `wasi-http` `proxy` world.""" def handle(self, request: IncomingRequest, response_out: ResponseOutparam) -> None: diff --git a/examples/matrix-math/app.py b/examples/matrix-math/app.py index c0824e62..5bee8c9f 100644 --- a/examples/matrix-math/app.py +++ b/examples/matrix-math/app.py @@ -3,22 +3,24 @@ import sys import numpy -import wit_world -from wit_world import exports +import wit +from wit.exports.wasi.cli_v0_2 import run, Run from componentize_py_types import Err -class WitWorld(wit_world.WitWorld): +@wit.guest +class MatrixMath(wit.WorldExports): def multiply(self, a: list[list[float]], b: list[list[float]]) -> list[list[float]]: print(f"matrix_multiply received arguments {a} and {b}") return numpy.matmul(a, b).tolist() # type: ignore -class Run(exports.Run): +@run.guest +class Cli(Run): def run(self) -> None: args = sys.argv[1:] if len(args) != 2: print("usage: matrix-math ", file=sys.stderr) exit(-1) - print(WitWorld().multiply(eval(args[0]), eval(args[1]))) + print(MatrixMath().multiply(eval(args[0]), eval(args[1]))) diff --git a/examples/sandbox/guest.py b/examples/sandbox/guest.py index 207925a5..a438caf6 100644 --- a/examples/sandbox/guest.py +++ b/examples/sandbox/guest.py @@ -1,4 +1,4 @@ -import wit_world +import wit from componentize_py_types import Err import json @@ -11,7 +11,8 @@ def handle(e: Exception) -> Err[str]: return Err(f"{type(e).__name__}: {message}") -class WitWorld(wit_world.WitWorld): +@wit.guest +class Sandbox(wit.WorldExports): def eval(self, expression: str) -> str: try: return json.dumps(eval(expression)) diff --git a/examples/tcp-p3/app.py b/examples/tcp-p3/app.py index 4b3a308b..f00e39fa 100644 --- a/examples/tcp-p3/app.py +++ b/examples/tcp-p3/app.py @@ -2,9 +2,9 @@ import asyncio import ipaddress from ipaddress import IPv4Address, IPv6Address -import wit_world -from wit_world import exports -from wit_world.imports.wasi_sockets_types import ( +import wit +from wit.exports.wasi.cli_v0_3 import run, Run +from wit.imports.wasi.sockets_v0_3.types import ( TcpSocket, IpSocketAddress_Ipv4, IpSocketAddress_Ipv6, @@ -17,7 +17,8 @@ IPAddress = IPv4Address | IPv6Address -class Run(exports.Run): +@run.guest +class Tcp(Run): async def run(self) -> None: args = sys.argv[1:] if len(args) != 1: @@ -67,7 +68,7 @@ async def send_and_receive(address: IPAddress, port: int) -> None: await sock.connect(make_socket_address(address, port)) - send_tx, send_rx = wit_world.byte_stream() + send_tx, send_rx = wit.byte_stream() async def write() -> None: await send_tx.write_all(b"hello, world!") diff --git a/examples/tcp/app.py b/examples/tcp/app.py index 00aae63f..3b692a6d 100644 --- a/examples/tcp/app.py +++ b/examples/tcp/app.py @@ -2,11 +2,12 @@ import asyncio import ipaddress from ipaddress import IPv4Address, IPv6Address -from wit_world import exports +from wit.exports.wasi.cli_v0_2 import run, Run from typing import Tuple -class Run(exports.Run): +@run.guest +class Tcp(Run): def run(self) -> None: args = sys.argv[1:] if len(args) != 1: diff --git a/src/command.rs b/src/command.rs index 8b018cb0..03774c5d 100644 --- a/src/command.rs +++ b/src/command.rs @@ -552,9 +552,8 @@ mod tests { r#" import bindings from bindings import x -from bindings.export import world_exports -@world_exports +@bindings.guest class Bindings(bindings.WorldExports): def y(self) -> None: x() @@ -600,18 +599,17 @@ world cli-world { &app_file, br#" import cli_world -from cli_world.export import world_exports -from cli_world.export.test import cli as export from cli_world.exports.test import cli as exports +from cli_world.exports.test.cli import cli_interface as cli_iface from cli_world.imports.test.cli import cli_interface from lib.wit.imports.test.lib import lib_interface -@world_exports +@cli_world.guest class CliWorld(cli_world.WorldExports): def foo(self) -> None: pass -@export.cli_interface +@cli_iface.guest class CliInterface(exports.CliInterface): def foo(self) -> None: lib_interface.foo() diff --git a/src/test.rs b/src/test.rs index 7e90aff7..942b7bd2 100644 --- a/src/test.rs +++ b/src/test.rs @@ -48,7 +48,7 @@ static ENGINE: Lazy = Lazy::new(|| { async fn make_component( wit: &str, worlds: &[&str], - world_module: Option<&str>, + bindings_module: Option<&str>, guest_code: &[(&str, &str)], python_path: &[&str], module_worlds: &[(&str, &[&str])], @@ -69,7 +69,7 @@ async fn make_component( worlds, features: &[], all_features: false, - world_module, + bindings_module, python_path: &python_path .iter() .copied() @@ -84,7 +84,6 @@ async fn make_component( stub_wasi: false, import_interface_names: &HashMap::new(), export_interface_names: &HashMap::new(), - full_names: false, intersect_world, } .generate() @@ -130,7 +129,7 @@ impl Tester { fn new( wit: &str, worlds: &[&str], - world_module: Option<&str>, + bindings_module: Option<&str>, guest_code: &[(&str, &str)], python_path: &[&str], module_worlds: &[(&str, &[&str])], @@ -145,7 +144,7 @@ impl Tester { let component = &Runtime::new()?.block_on(make_component( wit, worlds, - world_module, + bindings_module, guest_code, python_path, module_worlds, diff --git a/src/test/bar_sdk/componentize-py.toml b/src/test/bar_sdk/componentize-py.toml index 9c8f5b3f..89491857 100644 --- a/src/test/bar_sdk/componentize-py.toml +++ b/src/test/bar_sdk/componentize-py.toml @@ -1,3 +1,2 @@ wit_directory = "wit" bindings = "wit" -full_names = true diff --git a/src/test/echoes.rs b/src/test/echoes.rs index a651dc74..60a54fc8 100644 --- a/src/test/echoes.rs +++ b/src/test/echoes.rs @@ -216,9 +216,11 @@ impl super::Host for Host { const GUEST_CODE: &[(&str, &str)] = &[( "app.py", r#" -from echoes_test import exports -from echoes_test.imports import echoes +from echoes_test.exports.componentize_py import test as exports +from echoes_test.exports.componentize_py.test import echoes as echoes_exports +from echoes_test.imports.componentize_py.test import echoes +@echoes_exports.guest class Echoes(exports.Echoes): def echo_nothing(self): echoes.echo_nothing() diff --git a/src/test/python_source/app.py b/src/test/python_source/app.py index 7eded4e6..0c7dd3bb 100644 --- a/src/test/python_source/app.py +++ b/src/test/python_source/app.py @@ -1,6 +1,9 @@ import traceback import tests import resource_borrow_export +import resource_import_and_export +import resource_with_lists +import resource_floats_exports import resource_aggregates import resource_alias1 import resource_borrow_in_record @@ -8,46 +11,61 @@ import streams_and_futures as my_streams_and_futures from componentize_py_types import Result, Ok, Err -from tests import exports, imports -from tests.imports import resource_borrow_import -from tests.imports import simple_import_and_export -from tests.imports import simple_async_import_and_export -from tests.imports import host_thing_interface -from tests.exports import resource_alias2 -from tests.exports import streams_and_futures +from tests.exports.componentize_py import test as exports +from tests.imports.componentize_py import test as imports +from tests.imports.componentize_py.test import resource_borrow_import +from tests.imports.componentize_py.test import simple_import_and_export +from tests.imports.componentize_py.test import simple_async_import_and_export +from tests.imports.componentize_py.test import host_thing_interface +from tests.exports.componentize_py.test import resource_alias2 +from tests.exports.componentize_py.test import streams_and_futures from typing import Tuple, List, Optional -from foo_sdk.wit import exports as foo_exports -from foo_sdk.wit.imports.foo_interface import test as foo_test -from bar_sdk.wit import exports as bar_exports -from bar_sdk.wit.imports.foo_interface import test as bar_test - +from foo_sdk.wit.exports.foo import sdk as foo_exports +from foo_sdk.wit.exports.foo.sdk import foo_interface as foo_iface +from foo_sdk.wit.imports.foo.sdk.foo_interface import test as foo_test +from bar_sdk.wit.exports.bar import sdk as bar_exports +from bar_sdk.wit.exports.bar.sdk import bar_interface as bar_iface +from bar_sdk.wit.imports.foo.sdk.foo_interface import test as bar_test + +@exports.simple_export.guest class SimpleExport(exports.SimpleExport): def foo(self, v: int) -> int: return v + 3 +@exports.simple_import_and_export.guest class SimpleImportAndExport(exports.SimpleImportAndExport): def foo(self, v: int) -> int: return simple_import_and_export.foo(v) + 3 +@exports.simple_async_export.guest class SimpleAsyncExport(exports.SimpleAsyncExport): async def foo(self, v: int) -> int: return v + 3 +@exports.simple_async_import_and_export.guest class SimpleAsyncImportAndExport(exports.SimpleAsyncImportAndExport): async def foo(self, v: int) -> int: return (await simple_async_import_and_export.foo(v)) + 3 +@exports.resource_import_and_export.guest class ResourceImportAndExport(exports.ResourceImportAndExport): - pass + thing = resource_import_and_export.Thing +@exports.resource_borrow_export.guest class ResourceBorrowExport(exports.ResourceBorrowExport): + thing = resource_borrow_export.Thing + def foo(self, v: resource_borrow_export.Thing) -> int: return v.value + 2 +@exports.resource_with_lists.guest class ResourceWithLists(exports.ResourceWithLists): - pass + thing = resource_with_lists.Thing +@exports.resource_aggregates.guest class ResourceAggregates(exports.ResourceAggregates): + thing = resource_aggregates.Thing + def foo( self, r1: exports.resource_aggregates.R1, @@ -100,7 +118,10 @@ def foo( host_result2 ) + 4 +@exports.resource_alias1.guest class ResourceAlias1(exports.ResourceAlias1): + thing = resource_alias1.Thing + def a(self, f: exports.resource_alias1.Foo) -> List[resource_alias1.Thing]: return list( map( @@ -109,6 +130,7 @@ def a(self, f: exports.resource_alias1.Foo) -> List[resource_alias1.Thing]: ) ) +@exports.resource_alias2.guest class ResourceAlias2(exports.ResourceAlias2): def b(self, f: exports.resource_alias2.Foo, g: exports.resource_alias1.Foo) -> List[resource_alias1.Thing]: return list( @@ -121,7 +143,10 @@ def b(self, f: exports.resource_alias2.Foo, g: exports.resource_alias1.Foo) -> L ) ) +@exports.resource_borrow_in_record.guest class ResourceBorrowInRecord(exports.ResourceBorrowInRecord): + thing = resource_borrow_in_record.Thing + def test(self, a: List[exports.resource_borrow_in_record.Foo]) -> List[resource_borrow_in_record.Thing]: return list( map( @@ -186,7 +211,10 @@ async def write_host_thing(thing: host_thing_interface.HostThing, def unreachable() -> str: raise AssertionError +@exports.streams_and_futures.guest class StreamsAndFutures(exports.StreamsAndFutures): + thing = my_streams_and_futures.Thing + async def echo_stream_u8(self, stream: ByteStreamReader) -> ByteStreamReader: tx, rx = tests.byte_stream() componentize_py_async_support.spawn(pipe_bytes(stream, tx)) @@ -198,28 +226,29 @@ async def echo_future_string(self, future: FutureReader[str]) -> FutureReader[st return rx async def short_reads(self, stream: StreamReader[streams_and_futures.Thing]) -> StreamReader[streams_and_futures.Thing]: - tx, rx = tests.streams_and_futures_thing_stream() + tx, rx = tests.componentize_py_test_streams_and_futures_thing_stream() componentize_py_async_support.spawn(pipe_things(stream, tx)) return rx async def short_reads_host(self, stream: StreamReader[host_thing_interface.HostThing]) -> StreamReader[host_thing_interface.HostThing]: - tx, rx = tests.host_thing_interface_host_thing_stream() + tx, rx = tests.componentize_py_test_host_thing_interface_host_thing_stream() componentize_py_async_support.spawn(pipe_host_things(stream, tx)) return rx async def dropped_future_reader(self, value: str) -> tuple[FutureReader[streams_and_futures.Thing], FutureReader[streams_and_futures.Thing]]: - tx1, rx1 = tests.streams_and_futures_thing_future(unreachable) - tx2, rx2 = tests.streams_and_futures_thing_future(unreachable) + tx1, rx1 = tests.componentize_py_test_streams_and_futures_thing_future(unreachable) + tx2, rx2 = tests.componentize_py_test_streams_and_futures_thing_future(unreachable) componentize_py_async_support.spawn(write_thing(my_streams_and_futures.Thing(value), tx1, tx2)) return (rx1, rx2) async def dropped_future_reader_host(self, value: str) -> tuple[FutureReader[host_thing_interface.HostThing], FutureReader[host_thing_interface.HostThing]]: - tx1, rx1 = tests.host_thing_interface_host_thing_future(unreachable) - tx2, rx2 = tests.host_thing_interface_host_thing_future(unreachable) + tx1, rx1 = tests.componentize_py_test_host_thing_interface_host_thing_future(unreachable) + tx2, rx2 = tests.componentize_py_test_host_thing_interface_host_thing_future(unreachable) componentize_py_async_support.spawn(write_host_thing(host_thing_interface.HostThing(value), tx1, tx2)) return (rx1, rx2) -class Tests(tests.Tests): +@tests.guest +class Tests(tests.WorldExports): def test_resource_borrow_import(self, v: int) -> int: return resource_borrow_import.foo(resource_borrow_import.Thing(v + 1)) + 4 @@ -242,10 +271,12 @@ def test_refcounts(self): for _ in range(5 * 1024): chunk = tests.get_bytes(1024 * 1024) +@foo_iface.guest class FooInterface(foo_exports.FooInterface): def test(self, s: str) -> str: return foo_test(f"{s} FooInterface.test") -class BarSdkBarInterface(bar_exports.BarSdkBarInterface): +@bar_iface.guest +class BarInterface(bar_exports.BarInterface): def test(self, s: str) -> str: return bar_test(f"{s} BarInterface.test") diff --git a/src/test/python_source/resource_aggregates.py b/src/test/python_source/resource_aggregates.py index cd837596..20032294 100644 --- a/src/test/python_source/resource_aggregates.py +++ b/src/test/python_source/resource_aggregates.py @@ -1,5 +1,5 @@ -from tests.exports import resource_aggregates -from tests.imports.resource_aggregates import Thing as HostThing +from tests.exports.componentize_py.test import resource_aggregates +from tests.imports.componentize_py.test.resource_aggregates import Thing as HostThing class Thing(resource_aggregates.Thing): def __init__(self, v: int): diff --git a/src/test/python_source/resource_alias1.py b/src/test/python_source/resource_alias1.py index 586d0c40..3e8ce6ab 100644 --- a/src/test/python_source/resource_alias1.py +++ b/src/test/python_source/resource_alias1.py @@ -1,5 +1,5 @@ -from tests.exports import resource_alias1 -from tests.imports.resource_alias1 import Thing as HostThing +from tests.exports.componentize_py.test import resource_alias1 +from tests.imports.componentize_py.test.resource_alias1 import Thing as HostThing class Thing(resource_alias1.Thing): def __init__(self, v: str): diff --git a/src/test/python_source/resource_borrow_export.py b/src/test/python_source/resource_borrow_export.py index d2f46752..7b8b16cf 100644 --- a/src/test/python_source/resource_borrow_export.py +++ b/src/test/python_source/resource_borrow_export.py @@ -1,4 +1,4 @@ -from tests.exports import resource_borrow_export +from tests.exports.componentize_py.test import resource_borrow_export class Thing(resource_borrow_export.Thing): def __init__(self, v: int): diff --git a/src/test/python_source/resource_borrow_in_record.py b/src/test/python_source/resource_borrow_in_record.py index 249e14f2..b1ba4ba2 100644 --- a/src/test/python_source/resource_borrow_in_record.py +++ b/src/test/python_source/resource_borrow_in_record.py @@ -1,5 +1,5 @@ -from tests.exports import resource_borrow_in_record -from tests.imports.resource_borrow_in_record import Thing as HostThing +from tests.exports.componentize_py.test import resource_borrow_in_record +from tests.imports.componentize_py.test.resource_borrow_in_record import Thing as HostThing class Thing(resource_borrow_in_record.Thing): def __init__(self, v: str): diff --git a/src/test/python_source/resource_floats_exports.py b/src/test/python_source/resource_floats_exports.py index 7d582dcc..e127cfd2 100644 --- a/src/test/python_source/resource_floats_exports.py +++ b/src/test/python_source/resource_floats_exports.py @@ -11,4 +11,8 @@ def get(self) -> str: @staticmethod def add(a: Self, b: float) -> Self: - return Float(HostFloat.add(a.value, b).get() + 5) \ No newline at end of file + return Float(HostFloat.add(a.value, b).get() + 5) + +@resource_floats_exports.guest +class ResourceFloatsExports: + float = Float diff --git a/src/test/python_source/resource_import_and_export.py b/src/test/python_source/resource_import_and_export.py index 24825178..3bae5030 100644 --- a/src/test/python_source/resource_import_and_export.py +++ b/src/test/python_source/resource_import_and_export.py @@ -1,5 +1,5 @@ -from tests.exports import resource_import_and_export -from tests.imports.resource_import_and_export import Thing as HostThing +from tests.exports.componentize_py.test import resource_import_and_export +from tests.imports.componentize_py.test.resource_import_and_export import Thing as HostThing from typing import Self class Thing(resource_import_and_export.Thing): diff --git a/src/test/python_source/resource_with_lists.py b/src/test/python_source/resource_with_lists.py index 7c2ecf3e..81244fe5 100644 --- a/src/test/python_source/resource_with_lists.py +++ b/src/test/python_source/resource_with_lists.py @@ -1,5 +1,5 @@ -from tests.exports import resource_with_lists -from tests.imports.resource_with_lists import Thing as HostThing +from tests.exports.componentize_py.test import resource_with_lists +from tests.imports.componentize_py.test.resource_with_lists import Thing as HostThing from typing import List class Thing(resource_with_lists.Thing): diff --git a/src/test/python_source/streams_and_futures.py b/src/test/python_source/streams_and_futures.py index 4de18596..a2c522db 100644 --- a/src/test/python_source/streams_and_futures.py +++ b/src/test/python_source/streams_and_futures.py @@ -1,6 +1,6 @@ import tests -from tests.exports import streams_and_futures +from tests.exports.componentize_py.test import streams_and_futures class Thing(streams_and_futures.Thing): def __init__(self, v: str): diff --git a/src/test/tests.rs b/src/test/tests.rs index 8dd48bd3..d8537fd6 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -896,19 +896,21 @@ fn multiworld_intersect() -> Result<()> { &[( "app.py", r#" -from foo_sdk.wit import exports as foo_exports -from foo_sdk.wit.imports.foo_interface2 import test as foo_test2 +from foo_sdk.wit.exports.foo import sdk as foo_exports +from foo_sdk.wit.exports.foo.sdk import foo_interface as foo_iface +from foo_sdk.wit.imports.foo.sdk.foo_interface2 import test as foo_test2 try: - from foo_sdk.wit.imports.foo_interface import test as foo_test + from foo_sdk.wit.imports.foo.sdk.foo_interface import test as foo_test raise AssertionError except ModuleNotFoundError: pass try: - from bar_sdk.wit.imports.bar_interface import test as bar_test + from bar_sdk.wit.imports.bar.sdk.bar_interface import test as bar_test raise AssertionError except ModuleNotFoundError: pass +@foo_iface.guest class FooInterface(foo_exports.FooInterface): def test(self, s: str) -> str: return foo_test2(f"{s} FooInterface.test") diff --git a/test-generator/src/lib.rs b/test-generator/src/lib.rs index b58a191c..57d6e6c2 100644 --- a/test-generator/src/lib.rs +++ b/test-generator/src/lib.rs @@ -859,9 +859,11 @@ const GUEST_CODE: &[(&str, &str)] = &[ ( "app.py", r#" -from echoes_generated_test import exports -from echoes_generated_test.imports import echoes_generated +from echoes_generated_test.exports.componentize_py import test as exports +from echoes_generated_test.exports.componentize_py.test import echoes_generated as gen_exports +from echoes_generated_test.imports.componentize_py.test import echoes_generated +@gen_exports.guest class EchoesGenerated(exports.EchoesGenerated): {guest_functions} "#, diff --git a/tests/bindings.rs b/tests/bindings.rs index 9403ddea..4b7b8e11 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -22,7 +22,7 @@ fn lint_cli_bindings() -> anyhow::Result<()> { generate_bindings(&path, "wasi:cli/command@0.2.0")?; - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); mypy_check(&path, ["--strict", "-m", "app"]); @@ -41,7 +41,7 @@ fn lint_cli_p3_bindings() -> anyhow::Result<()> { generate_bindings(&path, "wasi:cli/command@0.3.0")?; - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); _ = dir.keep(); @@ -65,7 +65,7 @@ fn lint_http_bindings() -> anyhow::Result<()> { // poll_loop.py has many errors that might not be worth adjusting at the moment, so ignore for now fs::remove_file(path.join("poll_loop.py")).unwrap(); - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); mypy_check( &path, @@ -93,7 +93,7 @@ fn lint_http_p3_bindings() -> anyhow::Result<()> { generate_bindings(&path, "wasi:http/service@0.3.0")?; - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); _ = dir.keep(); @@ -116,7 +116,7 @@ fn lint_matrix_math_bindings() -> anyhow::Result<()> { generate_bindings(&path, "matrix-math")?; - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); mypy_check( &path, @@ -145,7 +145,7 @@ fn lint_sandbox_bindings() -> anyhow::Result<()> { .assert() .success(); - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); mypy_check(&path, ["--strict", "-m", "guest"]); @@ -164,7 +164,7 @@ fn lint_tcp_bindings() -> anyhow::Result<()> { generate_bindings(&path, "wasi:cli/command@0.2.0")?; - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); mypy_check(&path, ["--strict", "-m", "app"]); @@ -183,7 +183,7 @@ fn lint_tcp_p3_bindings() -> anyhow::Result<()> { generate_bindings(&path, "wasi:cli/command@0.3.0")?; - assert!(predicate::path::is_dir().eval(&path.join("wit_world"))); + assert!(predicate::path::is_dir().eval(&path.join("wit"))); mypy_check(&path, ["--strict", "-m", "app"]); @@ -219,7 +219,7 @@ world example { .assert() .success(); - assert!(predicate::path::is_dir().eval(&dir.path().join("wit_world"))); + assert!(predicate::path::is_dir().eval(&dir.path().join("wit"))); Command::new("python3") .current_dir(dir.path()) From 495d5eac20e25a6febb12e2735c3908168373b8a Mon Sep 17 00:00:00 2001 From: Mendy Berger <12537668+MendyBerger@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:05:04 -0400 Subject: [PATCH 3/4] Codegen update to #218 (new tests) --- src/command.rs | 815 ++++++++++++++++++++++++++++++++++++++++++++++ src/test/tests.rs | 370 +++++++++++++++++++++ 2 files changed, 1185 insertions(+) diff --git a/src/command.rs b/src/command.rs index 03774c5d..1ea5b267 100644 --- a/src/command.rs +++ b/src/command.rs @@ -686,4 +686,819 @@ world lib-world { }, ) } + + fn common(wit_path: PathBuf, world: &str) -> Common { + Common { + wit_path: vec![wit_path], + world: vec![world.to_owned()], + bindings_module: None, + quiet: false, + features: vec![], + all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), + } + } + + fn bindings_error(dir: &tempfile::TempDir, wit: &str, world: &str) -> String { + let wit_file = dir.path().join("test.wit"); + fs::write(&wit_file, wit).unwrap(); + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir).unwrap(); + + format!( + "{:?}", + generate_bindings( + common(wit_file, world), + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ) + } + + #[test] + fn function_named_guest_in_exported_interface_is_allowed() -> Result<()> { + // Exported functions become ABC methods and cannot clash with `guest`. + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +world w { + export bad: interface { + guest: func(); + } +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir, + }, + ) + } + + #[test] + fn reserved_guest_name_as_world_import() -> Result<()> { + let dir = tempfile::tempdir()?; + let error = bindings_error( + &dir, + r#" +package my:root; +world w { + import guest: func(); + export f: func(); +} +"#, + "w", + ); + + assert!(error.contains("reserved `guest` decorator"), "{error}"); + + Ok(()) + } + + #[test] + fn conflicting_interface_name_overrides() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +interface a { f: func(); } +interface b { g: func(); } +world w { + import a; + import b; +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.import_interface_name = vec![ + ("my:root/a".to_owned(), "dup".to_owned()), + ("my:root/b".to_owned(), "dup".to_owned()), + ]; + + let error = format!( + "{:?}", + generate_bindings( + common, + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!(error.contains("map to the module alias `dup`"), "{error}"); + + Ok(()) + } + + #[test] + fn interface_module_shadowed_by_package() -> Result<()> { + // `foo` would be both `imports/foo.py` and package `imports/foo/`. + let dir = tempfile::tempdir()?; + fs::create_dir_all(dir.path().join("wit/deps/dep"))?; + fs::write( + dir.path().join("wit/root.wit"), + r#" +package my:root; +world w { + import foo: interface { f: func(); } + import foo:bar/baz; +} +"#, + )?; + fs::write( + dir.path().join("wit/deps/dep/dep.wit"), + r#" +package foo:bar; +interface baz { g: func(); } +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let error = format!( + "{:?}", + generate_bindings( + common(dir.path().join("wit"), "w"), + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!( + error.contains("would be shadowed by the package"), + "{error}" + ); + + Ok(()) + } + + #[test] + fn canonical_version_collision_fallback() -> Result<()> { + // Same canonical version (`v0-1`) twice: fall back to full versions. + let dir = tempfile::tempdir()?; + fs::create_dir_all(dir.path().join("wit/deps/a"))?; + fs::create_dir_all(dir.path().join("wit/deps/b"))?; + fs::write( + dir.path().join("wit/root.wit"), + r#" +package my:root; +world w { + import foo:bar/baz@0.1.1; + import foo:bar/baz@0.1.2; +} +"#, + )?; + fs::write( + dir.path().join("wit/deps/a/a.wit"), + "package foo:bar@0.1.1;\ninterface baz { f: func(); }", + )?; + fs::write( + dir.path().join("wit/deps/b/b.wit"), + "package foo:bar@0.1.2;\ninterface baz { g: func(); }", + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(dir.path().join("wit"), "w"), + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + assert!(out_dir.join("wit/imports/foo/bar_v0_1_1/baz.py").is_file()); + assert!(out_dir.join("wit/imports/foo/bar_v0_1_2/baz.py").is_file()); + + Ok(()) + } + + #[test] + fn dotted_interface_name_override() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +interface a { f: func(); } +world w { + import a; +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.import_interface_name = + vec![("my:root/a".to_owned(), "my_pkg.my_module".to_owned())]; + + generate_bindings( + common, + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + assert!(out_dir.join("wit/imports/my_pkg/my_module.py").is_file()); + + Ok(()) + } + + #[test] + fn dotted_bindings_module() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +world w { + export f: func(); +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.bindings_module = Some("my.pkg".into()); + + generate_bindings( + common, + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + assert!(out_dir.join("my/__init__.py").is_file()); + assert!(out_dir.join("my/pkg/__init__.py").is_file()); + let generated = fs::read_to_string(out_dir.join("my/pkg/__init__.py"))?; + assert!(generated.contains("guest ="), "{generated}"); + + Ok(()) + } + + #[test] + fn helper_placeholders_follow_the_generated_paths() -> Result<()> { + // Both the module and the interface paths come from what was generated. + let dir = tempfile::tempdir()?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common("wit".into(), "wasi:http/proxy@0.2.0"); + common.bindings_module = Some("my.pkg".into()); + + generate_bindings( + common, + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + let generated = fs::read_to_string(out_dir.join("poll_loop.py"))?; + for expected in [ + "import my.pkg.imports.wasi.http_v0_2.types as types", + "import my.pkg.imports.wasi.io_v0_2.poll as poll", + "from my.pkg.imports.wasi.io_v0_2.streams import StreamError_Closed, InputStream", + ] { + assert!(generated.contains(expected), "{generated}"); + } + assert!(!generated.contains("WASI_HTTP_TYPES_MODULE"), "{generated}"); + + Ok(()) + } + + #[test] + fn helper_placeholders_kept_when_interfaces_are_absent() -> Result<()> { + // Nothing to point the helpers at, so leave the names greppable. + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +world w { + export f: func(); +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + let generated = fs::read_to_string(out_dir.join("poll_loop.py"))?; + assert!( + generated.contains("import WASI_HTTP_TYPES_MODULE as types"), + "{generated}" + ); + + Ok(()) + } + + #[test] + fn export_resource_classvar_uses_export_side() -> Result<()> { + // The resource `ClassVar` must reference the export-side class. + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +interface i { + resource r { + constructor(); + } + f: func(v: r); +} +world w { + import i; + export i; +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + let generated = fs::read_to_string(out_dir.join("wit/exports/my/root/__init__.py"))?; + assert!( + generated.contains("from .i import R as _my_root_i_r"), + "{generated}" + ); + assert!( + generated.contains("r: ClassVar[type[_my_root_i_r]]"), + "{generated}" + ); + // Packages expose their own submodules. + assert!(generated.contains("from . import i"), "{generated}"); + + Ok(()) + } + + #[test] + fn cross_direction_alias_collision() -> Result<()> { + // Flat aliases share one namespace across directions. + let dir = tempfile::tempdir()?; + let error = bindings_error( + &dir, + r#" +package my:root; +interface b { g: func(); } +world w { + import b; + export my-root-b: interface { + f: func(); + } +} +"#, + "w", + ); + + assert!(error.contains("module alias `my_root_b`"), "{error}"); + + Ok(()) + } + + #[test] + fn reserved_world_exports_name() -> Result<()> { + let dir = tempfile::tempdir()?; + let error = bindings_error( + &dir, + r#" +package my:root; +interface x { + record world-exports { x: u32 } +} +world w { + use x.{world-exports}; + export f: func(v: world-exports); +} +"#, + "w", + ); + + assert!(error.contains("reserved `WorldExports` class"), "{error}"); + + Ok(()) + } + + #[test] + fn cross_direction_version_fallback() -> Result<()> { + // Cross-direction canonical ties fall back instead of erroring. + let dir = tempfile::tempdir()?; + fs::create_dir_all(dir.path().join("wit/deps/a"))?; + fs::create_dir_all(dir.path().join("wit/deps/b"))?; + fs::write( + dir.path().join("wit/root.wit"), + r#" +package my:root; +world w { + import foo:bar/baz@0.1.1; + export foo:bar/baz@0.1.2; +} +"#, + )?; + fs::write( + dir.path().join("wit/deps/a/a.wit"), + "package foo:bar@0.1.1;\ninterface baz { f: func(); }", + )?; + fs::write( + dir.path().join("wit/deps/b/b.wit"), + "package foo:bar@0.1.2;\ninterface baz { g: func(); }", + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(dir.path().join("wit"), "w"), + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + assert!(out_dir.join("wit/imports/foo/bar_v0_1_1/baz.py").is_file()); + assert!(out_dir.join("wit/exports/foo/bar_v0_1_2/baz.py").is_file()); + + Ok(()) + } + + #[test] + fn reserved_guest_name_as_interface_alias() -> Result<()> { + let dir = tempfile::tempdir()?; + let error = bindings_error( + &dir, + r#" +package my:root; +world w { + import guest: interface { + f: func(); + } +} +"#, + "w", + ); + + assert!(error.contains("reserved `guest` decorator"), "{error}"); + + Ok(()) + } + + #[test] + fn invalid_interface_name_override() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +interface a { f: func(); } +world w { + import a; +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.import_interface_name = vec![("my:root/a".to_owned(), "1foo".to_owned())]; + + let error = format!( + "{:?}", + generate_bindings( + common, + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!(error.contains("not a valid Python identifier"), "{error}"); + + Ok(()) + } + + #[test] + fn invalid_bindings_module() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +world w { + export f: func(); +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.bindings_module = Some("my-module".into()); + + let error = format!( + "{:?}", + generate_bindings( + common, + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!(error.contains("not a valid Python module path"), "{error}"); + + Ok(()) + } + + #[test] + fn exported_interface_named_after_bindings_module() -> Result<()> { + let dir = tempfile::tempdir()?; + let error = bindings_error( + &dir, + r#" +package my:root; +world w { + export wit: interface { + f: func(); + } +} +"#, + "w", + ); + + assert!( + error.contains("collides with the bindings module name"), + "{error}" + ); + + Ok(()) + } + + #[test] + fn world_level_export_named_guest_is_allowed() -> Result<()> { + // Only world-level *imports* clash with the root `guest` decorator. + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + r#" +package my:root; +world w { + export guest: func(); +} +"#, + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir, + }, + ) + } + + #[test] + fn keyword_bindings_module() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write(&wit_file, "package my:root;\nworld w { export f: func(); }")?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.bindings_module = Some("class".into()); + + let error = format!( + "{:?}", + generate_bindings( + common, + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!(error.contains("not a valid Python module path"), "{error}"); + + Ok(()) + } + + #[test] + fn refuses_to_overwrite_wit_sources() -> Result<()> { + // The default module (`wit`) must not clobber a `wit/` source dir. + let dir = tempfile::tempdir()?; + fs::create_dir(dir.path().join("wit"))?; + fs::write( + dir.path().join("wit/app.wit"), + "package my:root;\nworld w { export f: func(); }", + )?; + + let error = format!( + "{:?}", + generate_bindings( + common(dir.path().join("wit"), "w"), + Bindings { + output_dir: dir.path().into(), + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!(error.contains("contains WIT source files"), "{error}"); + + Ok(()) + } + + #[test] + fn override_used_verbatim() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + "package my:root;\ninterface a { f: func(); }\nworld w { import a; }", + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.import_interface_name = vec![("my:root/a".to_owned(), "myPkg.myModule".to_owned())]; + + generate_bindings( + common, + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + assert!(out_dir.join("wit/imports/myPkg/myModule.py").is_file()); + + Ok(()) + } + + #[test] + fn rerun_into_same_directory() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + "package my:root;\ninterface a { f: func(); }\nworld w { import a; export a; }", + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(wit_file.clone(), "w"), + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + // Rename the interface and regenerate into the same directory. + fs::write( + &wit_file, + "package my:root;\ninterface b { f: func(); }\nworld w { import b; export b; }", + )?; + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + // No duplicate submodule imports, and no stale modules. + let generated = fs::read_to_string(out_dir.join("wit/imports/my/__init__.py"))?; + assert_eq!( + 1, + generated.matches("from . import root").count(), + "{generated}" + ); + assert!(!out_dir.join("wit/imports/my/root/a.py").exists()); + assert!(out_dir.join("wit/imports/my/root/b.py").is_file()); + + Ok(()) + } + + #[test] + fn refuses_to_overwrite_nested_wit_sources() -> Result<()> { + // WIT sources under `wit/deps/...` must be detected too. + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write(&wit_file, "package my:root;\nworld w { export f: func(); }")?; + let out_dir = dir.path().join("out"); + fs::create_dir_all(out_dir.join("wit/deps/dep"))?; + fs::write( + out_dir.join("wit/deps/dep/dep.wit"), + "package other:dep;\ninterface x { f: func(); }", + )?; + + let error = format!( + "{:?}", + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!(error.contains("contains WIT source files"), "{error}"); + + Ok(()) + } + + #[test] + fn world_import_named_exports_is_usable() -> Result<()> { + // The import shadows the subpackage attribute, not vice versa. + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + "package my:root;\ninterface i { g: func(); }\nworld w { import exports: func(); import i; export i; }", + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir.clone(), + }, + )?; + + let generated = fs::read_to_string(out_dir.join("wit/__init__.py"))?; + let bind = generated.find("from . import exports").unwrap(); + let def = generated.find("def exports").unwrap(); + assert!(bind < def, "{generated}"); + + Ok(()) + } + + #[test] + fn camel_case_export_override_rejected() -> Result<()> { + let dir = tempfile::tempdir()?; + let wit_file = dir.path().join("test.wit"); + fs::write( + &wit_file, + "package my:root;\ninterface a { f: func(); }\nworld w { export a; }", + )?; + let out_dir = dir.path().join("out"); + fs::create_dir(&out_dir)?; + + let mut common = common(wit_file, "w"); + common.export_interface_name = vec![("my:root/a".to_owned(), "Foo".to_owned())]; + + let error = format!( + "{:?}", + generate_bindings( + common, + Bindings { + output_dir: out_dir, + }, + ) + .expect_err("bindings generation should fail") + ); + + assert!(error.contains("abstract base class"), "{error}"); + + Ok(()) + } } diff --git a/src/test/tests.rs b/src/test/tests.rs index d8537fd6..03422027 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -964,6 +964,217 @@ class BarSdkBarInterface: Ok(()) } +/// Pins each helper import to the module owning that interface. +const HELPER_MODULE_ASSERTIONS: &str = r#" +import poll_loop + +for module, expected in ( + (poll_loop.types, "http_sdk.wit.imports.wasi.http_v0_2.types"), + (poll_loop.outgoing_handler, "http_sdk.wit.imports.wasi.http_v0_2.outgoing_handler"), + (poll_loop.streams, "http_sdk.wit.imports.wasi.io_v0_2.streams"), + (poll_loop.poll, "http_sdk.wit.imports.wasi.io_v0_2.poll"), +): + assert module.__name__ == expected, f"{module.__name__} != {expected}" + +from http_sdk.wit.exports.wasi.http_v0_2 import incoming_handler, IncomingHandler +from http_sdk.wit.imports.wasi.http_v0_2.types import IncomingRequest, ResponseOutparam + +@incoming_handler.guest +class Handler(IncomingHandler): + def handle(self, request: IncomingRequest, response_out: ResponseOutparam) -> None: + raise NotImplementedError +"#; + +/// Package with a `componentize-py.toml` owning `wit`, or the repo's WIT. +fn write_sdk(root: &std::path::Path, name: &str, wit: Option<&str>) -> Result<()> { + fn copy_dir(from: &std::path::Path, to: &std::path::Path) -> Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let to = to.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_dir(&entry.path(), &to)?; + } else { + std::fs::copy(entry.path(), to)?; + } + } + Ok(()) + } + + let dir = root.join(name); + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join("__init__.py"), "")?; + std::fs::write( + dir.join("componentize-py.toml"), + "wit_directory = \"wit\"\nbindings = \"wit\"\n", + )?; + + if let Some(wit) = wit { + std::fs::create_dir_all(dir.join("wit"))?; + std::fs::write(dir.join("wit/world.wit"), wit)?; + } else { + copy_dir(std::path::Path::new("wit"), &dir.join("wit"))?; + } + + Ok(()) +} + +fn componentize_app( + tempdir: &std::path::Path, + app_wit: &str, + worlds: &[&str], + module_worlds: &[(&str, &[&str])], + app: &str, +) -> Result<()> { + std::fs::write(tempdir.join("app.wit"), app_wit)?; + std::fs::write(tempdir.join("app.py"), app)?; + + tokio::runtime::Runtime::new()?.block_on( + crate::ComponentGenerator { + wit_paths: &[&tempdir.join("app.wit")], + worlds, + features: &[], + all_features: false, + bindings_module: None, + python_path: &[tempdir + .to_str() + .ok_or_else(|| anyhow!("unable to parse temporary directory path as UTF-8"))?], + module_worlds, + app_name: "app", + output_path: &tempdir.join("app.wasm"), + add_to_linker: None, + stub_wasi: false, + import_interface_names: &std::collections::HashMap::new(), + export_interface_names: &std::collections::HashMap::new(), + intersect_world: None, + } + .generate(), + ) +} + +/// Config-owned worlds only, so there is no synthesized module and no single +/// name to guess; `multiworld_intersect` never imports a helper. +#[test] +fn config_owned_worlds_resolve_bundled_helpers() -> Result<()> { + let tempdir = tempfile::tempdir()?; + write_sdk(tempdir.path(), "http_sdk", None)?; + write_sdk( + tempdir.path(), + "other_sdk", + Some( + "package other:sdk; +world other-world { + export g: func() -> string; +} +", + ), + )?; + + componentize_app( + tempdir.path(), + "package dummy:dummy;", + // No top-level world: `module_worlds` supplies both. + &[], + &[ + ("http_sdk", &["wasi:http/proxy@0.2.0"]), + ("other_sdk", &["other:sdk/other-world"]), + ], + &format!( + r#" +import other_sdk.wit as other + +@other.guest +class OtherWorld(other.WorldExports): + def g(self) -> str: + return "g" +{HELPER_MODULE_ASSERTIONS}"# + ), + ) +} + +/// The synthesized module is the tempting answer and the wrong one: only the +/// config-owned module holds the interfaces the helpers import. +#[test] +fn synthesized_world_alongside_config_owned_helpers() -> Result<()> { + let tempdir = tempfile::tempdir()?; + write_sdk(tempdir.path(), "http_sdk", None)?; + + componentize_app( + tempdir.path(), + "package dummy:dummy; +world dummy-world { + export f: func() -> string; +} +", + &["dummy:dummy/dummy-world"], + &[("http_sdk", &["wasi:http/proxy@0.2.0"])], + &format!( + r#" +import wit + +@wit.guest +class DummyWorld(wit.WorldExports): + def f(self) -> str: + return "f" +{HELPER_MODULE_ASSERTIONS}"# + ), + ) +} + +/// Two shapes `tests.wit` never produces: a shared two-way type with no +/// resource, and a resource-free exported interface reached via its package. +#[test] +fn exports_namespace_without_resources() -> Result<()> { + let tempdir = tempfile::tempdir()?; + + componentize_app( + tempdir.path(), + "package my:root; + +interface shared { + record point { x: u32, y: u32 } + swap: func(p: point) -> point; +} + +interface plain { + f: func() -> u32; +} + +world w { + import shared; + export shared; + export plain; +} +", + &["my:root/w"], + &[], + r#" +import typing +from wit.exports.my import root as root_exports +from wit.exports.my.root import shared +from wit.imports.my.root.shared import Point + +# `Point` has no two-way resource, so it exists only in the imports tree and +# both trees must name it there. +annotation = typing.get_type_hints(root_exports.Shared.swap)["p"] +assert annotation is Point, annotation +assert annotation.__module__ == "wit.imports.my.root.shared", annotation.__module__ + +@shared.guest +class Shared(root_exports.Shared): + def swap(self, p: Point) -> Point: + return Point(p.y, p.x) + +# `plain` has no resources, so nothing else pulls its submodule into the +# package; reaching it as an attribute is what the app is told to do. +@root_exports.plain.guest +class Plain(root_exports.Plain): + def f(self) -> int: + return 1 +"#, + ) +} + #[test] fn filesystem() -> Result<()> { let filename = "foo.txt"; @@ -1583,3 +1794,162 @@ fn test_dropped_future_reader_host(delay: bool) -> Result<()> { Ok(()) }) } + +/// Return `GUEST_CODE` with `app.py` patched via `edit`. +fn patched_guest_code(edit: impl Fn(&str) -> String) -> Vec<(&'static str, String)> { + GUEST_CODE + .iter() + .map(|&(name, content)| { + ( + name, + if name == "app.py" { + // Each patch matches on `\n`, which a CRLF checkout lacks. + let content = content.replace("\r\n", "\n"); + let patched = edit(&content); + assert_ne!(patched, content, "patch had no effect"); + patched + } else { + content.to_owned() + }, + ) + }) + .collect() +} + +fn patched_tester(edit: impl Fn(&str) -> String) -> Result> { + let code = patched_guest_code(edit); + let code = code + .iter() + .map(|(name, content)| (*name, content.as_str())) + .collect::>(); + + Tester::::new( + include_str!("wit/tests.wit"), + &["componentize-py:test/tests"], + Some("tests"), + &code, + &["src/test"], + &[ + ("foo_sdk", &["foo:sdk/foo-world"]), + ("bar_sdk", &["bar:sdk/bar-world"]), + ], + None, + *SEED, + ) +} + +#[test] +fn missing_guest_registration() { + let result = patched_tester(|code| code.replace("@exports.simple_export.guest\n", "")); + + assert!(matches!(result, Err(error) if format!("{error}").contains( + "no implementation registered for `componentize-py:test/simple-export`" + ))); +} + +#[test] +fn duplicate_guest_registration() { + let result = patched_tester(|code| { + format!( + "{code} +@exports.simple_export.guest +class AnotherSimpleExport: + def foo(self, v: int) -> int: + return v +" + ) + }); + + assert!(matches!(result, Err(error) if format!("{error}").contains( + "multiple implementations registered for `componentize-py:test/simple-export`" + ))); +} + +#[test] +fn missing_resource_attribute() { + let result = patched_tester(|code| { + code.replace( + "class ResourceAggregates(exports.ResourceAggregates):\n thing = resource_aggregates.Thing\n", + "class ResourceAggregates(exports.ResourceAggregates):\n", + ) + }); + + assert!(matches!(result, Err(error) if format!("{error}").contains( + "must declare a class attribute `thing`" + ))); +} + +#[test] +fn guest_without_abc_subclass() -> Result<()> { + // Subclassing the generated abstract base class is optional; the `guest` + // decorator alone is enough. + let tester = patched_tester(|code| { + code.replace( + "class SimpleExport(exports.SimpleExport):", + "class SimpleExport:", + ) + })?; + + tester.test(|world, store, runtime| { + assert_eq!( + 42 + 3, + runtime.block_on( + world + .componentize_py_test_simple_export() + .call_foo(store, 42) + )? + ); + + Ok(()) + }) +} + +#[test] +fn guest_instance_shared_across_functions() -> Result<()> { + // All functions of an interface (or world) dispatch to one shared instance + // of the registered class. + let tester = patched_tester(|code| { + code.replace( + " def test_resource_borrow_import(self, v: int) -> int:\n", + " def __init__(self) -> None:\n self.stash = b\"unset\"\n\n \ + def test_resource_borrow_import(self, v: int) -> int:\n \ + self.stash = str(v).encode()\n", + ) + .replace( + " def read_file(self, path: str) -> bytes:\n try:\n \ + with open(file=path, mode=\"rb\") as f:\n return f.read()\n \ + except:\n raise Err(traceback.format_exc())\n", + " def read_file(self, path: str) -> bytes:\n return self.stash\n", + ) + })?; + + tester.test(|world, store, runtime| { + runtime.block_on(async { + world + .call_test_resource_borrow_import(&mut *store, 42) + .await?; + + let value = world + .call_read_file(&mut *store, "unused") + .await? + .map_err(|s| anyhow!("{s}"))?; + + assert_eq!(b"42".as_slice(), &value); + + Ok(()) + }) + }) +} + +#[test] +fn guest_resource_attribute_inherited() -> Result<()> { + // Resource declarations may come from a base class. + patched_tester(|code| { + code.replace( + "@exports.resource_aggregates.guest\nclass ResourceAggregates(exports.ResourceAggregates):\n thing = resource_aggregates.Thing\n", + "class _AggregatesBase:\n thing = resource_aggregates.Thing\n\n@exports.resource_aggregates.guest\nclass ResourceAggregates(_AggregatesBase, exports.ResourceAggregates):\n", + ) + })?; + + Ok(()) +} From a0828b8335e72c422617a4dcf40f5173d5211a1b Mon Sep 17 00:00:00 2001 From: Mendy Berger <12537668+MendyBerger@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:31:26 -0400 Subject: [PATCH 4/4] Incorporate feedback from review --- src/command.rs | 222 ++++++++++++++-------------------------------- src/lib.rs | 36 +++----- src/python.rs | 4 +- src/test/tests.rs | 57 +++++++++--- 4 files changed, 123 insertions(+), 196 deletions(-) diff --git a/src/command.rs b/src/command.rs index 1ea5b267..8e5438dc 100644 --- a/src/command.rs +++ b/src/command.rs @@ -205,6 +205,13 @@ pub struct Bindings { /// /// This will be created if it does not already exist. pub output_dir: PathBuf, + + /// Generate into the bindings module directory even if it already exists. + /// + /// Nothing is deleted, so bindings from a previous run may be left behind + /// alongside the new ones. + #[arg(long)] + pub allow_existing: bool, } fn parse_key_value(s: &str) -> Result<(String, String), String> { @@ -250,6 +257,7 @@ fn generate_bindings(common: Common, bindings: Bindings) -> Result<()> { all_features: common.all_features, bindings_module: common.bindings_module.as_deref(), output_dir: &bindings.output_dir, + allow_existing: bindings.allow_existing, import_interface_names: &common .import_interface_name .iter() @@ -454,9 +462,7 @@ mod tests { import_interface_name: Vec::new(), export_interface_name: Vec::new(), }; - let bindings = Bindings { - output_dir: out_dir.path().into(), - }; + let bindings = bindings(out_dir.path()); generate_bindings(common, bindings)?; // Then the gated feature doesn't appear @@ -484,9 +490,7 @@ mod tests { import_interface_name: Vec::new(), export_interface_name: Vec::new(), }; - let bindings = Bindings { - output_dir: out_dir.path().into(), - }; + let bindings = bindings(out_dir.path()); generate_bindings(common, bindings)?; // Then the gated feature doesn't appear @@ -514,9 +518,7 @@ mod tests { import_interface_name: Vec::new(), export_interface_name: Vec::new(), }; - let bindings = Bindings { - output_dir: out_dir.path().into(), - }; + let bindings = bindings(out_dir.path()); generate_bindings(common, bindings)?; // Then the gated feature doesn't appear @@ -543,9 +545,7 @@ mod tests { import_interface_name: Vec::new(), export_interface_name: Vec::new(), }; - let bindings = Bindings { - output_dir: out_dir.path().into(), - }; + let bindings = bindings(out_dir.path()); generate_bindings(common.clone(), bindings)?; fs::write( out_dir.path().join("app.py"), @@ -655,9 +655,7 @@ world lib-world { import_interface_name: Vec::new(), export_interface_name: Vec::new(), }, - Bindings { - output_dir: lib_wit_dir, - }, + bindings(lib_wit_dir), )?; componentize( @@ -687,6 +685,13 @@ world lib-world { ) } + fn bindings(output_dir: impl Into) -> Bindings { + Bindings { + output_dir: output_dir.into(), + allow_existing: false, + } + } + fn common(wit_path: PathBuf, world: &str) -> Common { Common { wit_path: vec![wit_path], @@ -708,13 +713,8 @@ world lib-world { format!( "{:?}", - generate_bindings( - common(wit_file, world), - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common(wit_file, world), bindings(out_dir),) + .expect_err("bindings generation should fail") ) } @@ -737,12 +737,7 @@ world w { let out_dir = dir.path().join("out"); fs::create_dir(&out_dir)?; - generate_bindings( - common(wit_file, "w"), - Bindings { - output_dir: out_dir, - }, - ) + generate_bindings(common(wit_file, "w"), bindings(out_dir)) } #[test] @@ -792,13 +787,8 @@ world w { let error = format!( "{:?}", - generate_bindings( - common, - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common, bindings(out_dir),) + .expect_err("bindings generation should fail") ); assert!(error.contains("map to the module alias `dup`"), "{error}"); @@ -833,13 +823,8 @@ interface baz { g: func(); } let error = format!( "{:?}", - generate_bindings( - common(dir.path().join("wit"), "w"), - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common(dir.path().join("wit"), "w"), bindings(out_dir),) + .expect_err("bindings generation should fail") ); assert!( @@ -879,9 +864,7 @@ world w { generate_bindings( common(dir.path().join("wit"), "w"), - Bindings { - output_dir: out_dir.clone(), - }, + bindings(out_dir.clone()), )?; assert!(out_dir.join("wit/imports/foo/bar_v0_1_1/baz.py").is_file()); @@ -911,12 +894,7 @@ world w { common.import_interface_name = vec![("my:root/a".to_owned(), "my_pkg.my_module".to_owned())]; - generate_bindings( - common, - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common, bindings(out_dir.clone()))?; assert!(out_dir.join("wit/imports/my_pkg/my_module.py").is_file()); @@ -942,12 +920,7 @@ world w { let mut common = common(wit_file, "w"); common.bindings_module = Some("my.pkg".into()); - generate_bindings( - common, - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common, bindings(out_dir.clone()))?; assert!(out_dir.join("my/__init__.py").is_file()); assert!(out_dir.join("my/pkg/__init__.py").is_file()); @@ -967,12 +940,7 @@ world w { let mut common = common("wit".into(), "wasi:http/proxy@0.2.0"); common.bindings_module = Some("my.pkg".into()); - generate_bindings( - common, - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common, bindings(out_dir.clone()))?; let generated = fs::read_to_string(out_dir.join("poll_loop.py"))?; for expected in [ @@ -1004,12 +972,7 @@ world w { let out_dir = dir.path().join("out"); fs::create_dir(&out_dir)?; - generate_bindings( - common(wit_file, "w"), - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common(wit_file, "w"), bindings(out_dir.clone()))?; let generated = fs::read_to_string(out_dir.join("poll_loop.py"))?; assert!( @@ -1044,12 +1007,7 @@ world w { let out_dir = dir.path().join("out"); fs::create_dir(&out_dir)?; - generate_bindings( - common(wit_file, "w"), - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common(wit_file, "w"), bindings(out_dir.clone()))?; let generated = fs::read_to_string(out_dir.join("wit/exports/my/root/__init__.py"))?; assert!( @@ -1142,9 +1100,7 @@ world w { generate_bindings( common(dir.path().join("wit"), "w"), - Bindings { - output_dir: out_dir.clone(), - }, + bindings(out_dir.clone()), )?; assert!(out_dir.join("wit/imports/foo/bar_v0_1_1/baz.py").is_file()); @@ -1196,13 +1152,8 @@ world w { let error = format!( "{:?}", - generate_bindings( - common, - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common, bindings(out_dir),) + .expect_err("bindings generation should fail") ); assert!(error.contains("not a valid Python identifier"), "{error}"); @@ -1231,13 +1182,8 @@ world w { let error = format!( "{:?}", - generate_bindings( - common, - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common, bindings(out_dir),) + .expect_err("bindings generation should fail") ); assert!(error.contains("not a valid Python module path"), "{error}"); @@ -1286,12 +1232,7 @@ world w { let out_dir = dir.path().join("out"); fs::create_dir(&out_dir)?; - generate_bindings( - common(wit_file, "w"), - Bindings { - output_dir: out_dir, - }, - ) + generate_bindings(common(wit_file, "w"), bindings(out_dir)) } #[test] @@ -1307,13 +1248,8 @@ world w { let error = format!( "{:?}", - generate_bindings( - common, - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common, bindings(out_dir),) + .expect_err("bindings generation should fail") ); assert!(error.contains("not a valid Python module path"), "{error}"); @@ -1322,8 +1258,7 @@ world w { } #[test] - fn refuses_to_overwrite_wit_sources() -> Result<()> { - // The default module (`wit`) must not clobber a `wit/` source dir. + fn refuses_to_write_into_existing_directory() -> Result<()> { let dir = tempfile::tempdir()?; fs::create_dir(dir.path().join("wit"))?; fs::write( @@ -1333,16 +1268,13 @@ world w { let error = format!( "{:?}", - generate_bindings( - common(dir.path().join("wit"), "w"), - Bindings { - output_dir: dir.path().into(), - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common(dir.path().join("wit"), "w"), bindings(dir.path())) + .expect_err("bindings generation should fail") ); - assert!(error.contains("contains WIT source files"), "{error}"); + assert!(error.contains("already exists"), "{error}"); + // Nothing was touched. + assert!(dir.path().join("wit/app.wit").is_file()); Ok(()) } @@ -1361,12 +1293,7 @@ world w { let mut common = common(wit_file, "w"); common.import_interface_name = vec![("my:root/a".to_owned(), "myPkg.myModule".to_owned())]; - generate_bindings( - common, - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common, bindings(out_dir.clone()))?; assert!(out_dir.join("wit/imports/myPkg/myModule.py").is_file()); @@ -1384,12 +1311,7 @@ world w { let out_dir = dir.path().join("out"); fs::create_dir(&out_dir)?; - generate_bindings( - common(wit_file.clone(), "w"), - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common(wit_file.clone(), "w"), bindings(out_dir.clone()))?; // Rename the interface and regenerate into the same directory. fs::write( @@ -1400,25 +1322,27 @@ world w { common(wit_file, "w"), Bindings { output_dir: out_dir.clone(), + allow_existing: true, }, )?; - // No duplicate submodule imports, and no stale modules. + // No duplicate submodule imports. The renamed module is left behind, + // which is why `--allow-existing` has to be asked for. let generated = fs::read_to_string(out_dir.join("wit/imports/my/__init__.py"))?; assert_eq!( 1, generated.matches("from . import root").count(), "{generated}" ); - assert!(!out_dir.join("wit/imports/my/root/a.py").exists()); + assert!(out_dir.join("wit/imports/my/root/a.py").is_file()); assert!(out_dir.join("wit/imports/my/root/b.py").is_file()); Ok(()) } #[test] - fn refuses_to_overwrite_nested_wit_sources() -> Result<()> { - // WIT sources under `wit/deps/...` must be detected too. + fn allow_existing_writes_alongside_wit_sources() -> Result<()> { + // A WIT directory may legitimately hold generated Python too. let dir = tempfile::tempdir()?; let wit_file = dir.path().join("test.wit"); fs::write(&wit_file, "package my:root;\nworld w { export f: func(); }")?; @@ -1429,18 +1353,16 @@ world w { "package other:dep;\ninterface x { f: func(); }", )?; - let error = format!( - "{:?}", - generate_bindings( - common(wit_file, "w"), - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") - ); + generate_bindings( + common(wit_file, "w"), + Bindings { + output_dir: out_dir.clone(), + allow_existing: true, + }, + )?; - assert!(error.contains("contains WIT source files"), "{error}"); + assert!(out_dir.join("wit/__init__.py").is_file()); + assert!(out_dir.join("wit/deps/dep/dep.wit").is_file()); Ok(()) } @@ -1457,12 +1379,7 @@ world w { let out_dir = dir.path().join("out"); fs::create_dir(&out_dir)?; - generate_bindings( - common(wit_file, "w"), - Bindings { - output_dir: out_dir.clone(), - }, - )?; + generate_bindings(common(wit_file, "w"), bindings(out_dir.clone()))?; let generated = fs::read_to_string(out_dir.join("wit/__init__.py"))?; let bind = generated.find("from . import exports").unwrap(); @@ -1488,13 +1405,8 @@ world w { let error = format!( "{:?}", - generate_bindings( - common, - Bindings { - output_dir: out_dir, - }, - ) - .expect_err("bindings generation should fail") + generate_bindings(common, bindings(out_dir),) + .expect_err("bindings generation should fail") ); assert!(error.contains("abstract base class"), "{error}"); diff --git a/src/lib.rs b/src/lib.rs index e1ad3a1e..a3b7dae4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -183,6 +183,8 @@ pub struct BindingsGenerator<'a> { pub output_dir: &'a Path, pub import_interface_names: &'a HashMap<&'a str, &'a str>, pub export_interface_names: &'a HashMap<&'a str, &'a str>, + /// Write into `output_dir` even if the bindings module directory exists. + pub allow_existing: bool, } impl BindingsGenerator<'_> { @@ -240,17 +242,14 @@ impl BindingsGenerator<'_> { let bindings_module = self.bindings_module.unwrap_or(DEFAULT_BINDINGS_MODULE); validate_bindings_module(bindings_module)?; let world_dir = self.output_dir.join(bindings_module.replace('.', "/")); - // A `wit/` WIT-source directory clashes with the default module name. - if world_dir.is_dir() { - if contains_wit_files(&world_dir)? { - bail!( - "refusing to write bindings into {}, which contains WIT source files; \ - specify a different output directory or `--bindings-module`", - world_dir.display() - ); - } - fs::remove_dir_all(&world_dir)?; - } + // Writing into an existing directory can leave bindings from a previous + // run behind, so make the caller opt in rather than deleting anything. + ensure!( + self.allow_existing || !world_dir.exists(), + "{} already exists; remove it, or pass `--allow-existing` to write into it \ + (bindings from a previous run are not removed)", + world_dir.display() + ); fs::create_dir_all(&world_dir)?; create_module_ancestors(self.output_dir, bindings_module)?; let mut locations = Locations::default(); @@ -322,21 +321,6 @@ pub(crate) fn resolve_deprecated( Ok(bindings_module.or(world_module)) } -fn contains_wit_files(dir: &Path) -> Result { - for entry in fs::read_dir(dir)? { - let path = entry?.path(); - if path.is_dir() { - if contains_wit_files(&path)? { - return Ok(true); - } - } else if path.extension().is_some_and(|ext| ext == "wit") { - return Ok(true); - } - } - - Ok(false) -} - /// Reject `--bindings-module` values which are not importable Python paths. fn validate_bindings_module(module: &str) -> Result<()> { if module diff --git a/src/python.rs b/src/python.rs index 9beca41a..c05ef12f 100644 --- a/src/python.rs +++ b/src/python.rs @@ -82,7 +82,7 @@ fn python_componentize( #[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "generate_bindings")] -#[pyo3(signature = (wit_path, worlds, features, all_features, bindings_module, output_dir, import_interface_names, export_interface_names, full_names = None, world_module = None))] +#[pyo3(signature = (wit_path, worlds, features, all_features, bindings_module, output_dir, import_interface_names, export_interface_names, allow_existing = false, full_names = None, world_module = None))] fn python_generate_bindings( wit_path: Vec, worlds: Vec, @@ -92,6 +92,7 @@ fn python_generate_bindings( output_dir: PathBuf, import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, + allow_existing: bool, full_names: Option, world_module: Option<&str>, ) -> PyResult<()> { @@ -109,6 +110,7 @@ fn python_generate_bindings( all_features, bindings_module: bindings_module.as_deref(), output_dir: &output_dir, + allow_existing, import_interface_names: &import_interface_names .iter() .map(|(a, b)| (a.as_ref(), b.as_ref())) diff --git a/src/test/tests.rs b/src/test/tests.rs index 03422027..c5bee0ab 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -1006,7 +1006,9 @@ fn write_sdk(root: &std::path::Path, name: &str, wit: Option<&str>) -> Result<() std::fs::write(dir.join("__init__.py"), "")?; std::fs::write( dir.join("componentize-py.toml"), - "wit_directory = \"wit\"\nbindings = \"wit\"\n", + r#"wit_directory = "wit" +bindings = "wit" +"#, )?; if let Some(wit) = wit { @@ -1840,7 +1842,13 @@ fn patched_tester(edit: impl Fn(&str) -> String) -> Result> { #[test] fn missing_guest_registration() { - let result = patched_tester(|code| code.replace("@exports.simple_export.guest\n", "")); + let result = patched_tester(|code| { + code.replace( + r#"@exports.simple_export.guest +"#, + "", + ) + }); assert!(matches!(result, Err(error) if format!("{error}").contains( "no implementation registered for `componentize-py:test/simple-export`" @@ -1869,8 +1877,11 @@ class AnotherSimpleExport: fn missing_resource_attribute() { let result = patched_tester(|code| { code.replace( - "class ResourceAggregates(exports.ResourceAggregates):\n thing = resource_aggregates.Thing\n", - "class ResourceAggregates(exports.ResourceAggregates):\n", + r#"class ResourceAggregates(exports.ResourceAggregates): + thing = resource_aggregates.Thing +"#, + r#"class ResourceAggregates(exports.ResourceAggregates): +"#, ) }); @@ -1910,16 +1921,26 @@ fn guest_instance_shared_across_functions() -> Result<()> { // of the registered class. let tester = patched_tester(|code| { code.replace( - " def test_resource_borrow_import(self, v: int) -> int:\n", - " def __init__(self) -> None:\n self.stash = b\"unset\"\n\n \ - def test_resource_borrow_import(self, v: int) -> int:\n \ - self.stash = str(v).encode()\n", + r#" def test_resource_borrow_import(self, v: int) -> int: +"#, + r#" def __init__(self) -> None: + self.stash = b"unset" + + def test_resource_borrow_import(self, v: int) -> int: + self.stash = str(v).encode() +"#, ) .replace( - " def read_file(self, path: str) -> bytes:\n try:\n \ - with open(file=path, mode=\"rb\") as f:\n return f.read()\n \ - except:\n raise Err(traceback.format_exc())\n", - " def read_file(self, path: str) -> bytes:\n return self.stash\n", + r#" def read_file(self, path: str) -> bytes: + try: + with open(file=path, mode="rb") as f: + return f.read() + except: + raise Err(traceback.format_exc()) +"#, + r#" def read_file(self, path: str) -> bytes: + return self.stash +"#, ) })?; @@ -1946,8 +1967,16 @@ fn guest_resource_attribute_inherited() -> Result<()> { // Resource declarations may come from a base class. patched_tester(|code| { code.replace( - "@exports.resource_aggregates.guest\nclass ResourceAggregates(exports.ResourceAggregates):\n thing = resource_aggregates.Thing\n", - "class _AggregatesBase:\n thing = resource_aggregates.Thing\n\n@exports.resource_aggregates.guest\nclass ResourceAggregates(_AggregatesBase, exports.ResourceAggregates):\n", + r#"@exports.resource_aggregates.guest +class ResourceAggregates(exports.ResourceAggregates): + thing = resource_aggregates.Thing +"#, + r#"class _AggregatesBase: + thing = resource_aggregates.Thing + +@exports.resource_aggregates.guest +class ResourceAggregates(_AggregatesBase, exports.ResourceAggregates): +"#, ) })?;