Proof of concept of bindgen tests using componentize-py (bytecodealliance#234)

* Proof of concept of bindgen tests using componentize-py

The basic idea is that you define a wit file along with guest code
in Python.  We can then generate a wasm component using componentize-py,
host bindings using `wasmtime.bindgen`, and then test that the generated
bindings work as expected.

* Restructure bindgen testcases

* Each test case is now in the same directory with the corresponding
  WIT file and guest code (app.py).
* The helper functions for generating wasm components and host bindings
  is moved to a pytest fixture to avoid relative package issues when
  using direct imports.
* Remove the use of changing the cwd in the test process.

Author: jamesls

.gitignore

@@ -14,3 +14,4 @@ wasmtime/win32-*
 wasmtime/include
 wasmtime/bindgen/generated
 tests/codegen/generated
+tests/bindgen/generated

mypy.ini

@@ -12,6 +12,9 @@ disallow_untyped_calls = True
 disallow_untyped_defs = True
 disallow_incomplete_defs = True
 
+[mypy-componentize_py.*]
+ignore_missing_imports = True
+
 [mypy-tests.*]
 check_untyped_defs = True
 

pytest.ini

@@ -1,2 +1,4 @@
 [pytest]
-addopts = --doctest-modules --mypy
+addopts = --doctest-modules --mypy --ignore-glob=tests/bindgen/*/app.py
+norecursedirs = 
+    tests/bindgen/generated/*

setup.py

@@ -42,6 +42,7 @@
             'pytest',
             'pycparser',
             'pytest-mypy',
+            'componentize-py',
         ],
     },
     classifiers=[

tests/bindgen/__init__.py

@@ -0,0 +1,3 @@
+class Barefuncs:
+    def foo(self, a):
+        return a + 1

tests/bindgen/bare_funcs/app.py

@@ -0,0 +1,5 @@
+package component:barefuncs;
+
+world barefuncs {
+  export foo: func(a: s32) -> s32;
+}

tests/bindgen/bare_funcs/component.wit

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+
+def test_bare_funcs(bindgen_testcase):
+    store, root = bindgen_testcase(
+        guest_code_dir=Path(__file__).parent,
+        world_name='barefuncs',
+    )
+    assert root.foo(store, 10) == 11

tests/bindgen/bare_funcs/test_bare_funcs.py

@@ -0,0 +1,146 @@
+"""Fixtures to define test suites for generated code Python guest code.
+
+These tests work by allowing you to write a WIT file, implement the guest
+code in Python via componentize-py, and then test the generated Python
+bindings. To add a new test, first create the needed fixtures:
+
+* Create a new sub directory.
+* Within that directory create a `.wit` file.
+* Create an `app.py` file in that directory implementing the guest code.
+
+Then to write the test itself:
+
+* Create a `test_<name>.py` in the same directory.
+* Use the `bindgest_testcase` in your test to create the wasm component
+  and generate python bindings for this component.
+
+## Example
+
+Given this directory:
+
+```
+bare_funcs/
+├── app.py          <-- guest code implementation
+├── barefuncs       <-- componentize-py bindings
+│   ├── __init__.py
+│   └── types.py
+├── component.wit   <-- test .wit file
+└── test_mycomp.py  <-- pytest test case of bindings
+```
+
+With a `component.wit` file of:
+
+```wit
+package component:barefuncs;
+
+world barefuncs {
+  export foo: func(a: s32) -> s32;
+}
+```
+
+And guest code of:
+
+```python
+class Barefuncs:
+    def foo(self, a: int) -> int:
+        return a + 1
+```
+
+You can write a testcase for this using:
+
+```python
+from pathlib import Path
+
+
+def test_bare_funcs(bindgen_testcase):
+    testcase = bindgen_testcase(
+        guest_code_dir=Path(__file__).parent,
+        world_name='barefuncs',
+    )
+    store, root = generate_bindings(testcase)
+    assert root.foo(store, 10) == 11
+```
+
+"""
+from pathlib import Path
+from dataclasses import dataclass, field
+import importlib
+import tempfile
+import subprocess
+import shutil
+
+from pytest import fixture
+
+import wasmtime
+from wasmtime.bindgen import generate
+
+
+TEST_ROOT = Path(__file__).parent
+BINDGEN_DIR = TEST_ROOT / 'generated'
+
+
+@dataclass
+class BindgenTestCase:
+    guest_code_dir: Path
+    world_name: str
+    wit_filename: str = 'component.wit'
+    app_dir: Path = field(init=False)
+    app_name: str = field(init=False, default='app', repr=False)
+
+    def __post_init__(self):
+        self.app_dir = Path(self.guest_code_dir).resolve()
+
+    @property
+    def wit_full_path(self):
+        return self.guest_code_dir.joinpath(self.wit_filename)
+
+    @property
+    def testsuite_name(self):
+        # The name of the directory that contains the
+        # guest Python code is used as the identifier for
+        # package names, etc.
+        return self.guest_code_dir.name
+
+
+def generate_bindings(guest_code_dir: Path,
+                      world_name: str,
+                      wit_filename: str = 'component.wit'):
+    tc = BindgenTestCase(
+            guest_code_dir=guest_code_dir,
+            world_name=world_name,
+            wit_filename=wit_filename)
+    return _generate_bindings(tc)
+
+
+def _generate_bindings(testcase: BindgenTestCase):
+    wit_path = testcase.wit_full_path
+    componentize_py = shutil.which('componentize-py')
+    if componentize_py is None:
+        raise RuntimeError("Could not find componentize-py executable.")
+    with tempfile.NamedTemporaryFile('w') as f:
+        output_wasm = str(f.name + '.wasm')
+        subprocess.run([
+            componentize_py, '-d', str(wit_path), '-w', testcase.world_name,
+            'componentize', '--stub-wasi', testcase.app_name,
+            '-o', output_wasm
+        ], check=True, cwd=testcase.guest_code_dir)
+        # Once we've done that now generate the python bindings.
+        testsuite_name = testcase.testsuite_name
+        with open(output_wasm, 'rb') as out:
+            # Mapping of filename -> content_bytes
+            results = generate(testsuite_name, out.read())
+        for filename, contents in results.items():
+            path = BINDGEN_DIR / testsuite_name / filename
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(contents)
+    # Return an instantiated module for the caller to test.
+    pkg = importlib.import_module(f'.generated.{testsuite_name}',
+                                  package=__package__)
+    store = wasmtime.Store()
+    root = pkg.Root(store)
+    return store, root
+
+
+@fixture
+def bindgen_testcase():
+    return generate_bindings

tests/bindgen/conftest.py

@@ -0,0 +1,30 @@
+import sys
+from types import ModuleType
+
+
+class MyInterfaceName:
+    def interface_func(self, foo: str) -> str:
+        return f"hello {foo}"
+
+
+# componentize-py expects that resources within an interface are defined
+# as a class in a separate module that matches the interface name.
+#
+# Normally, you'd want to go the more typical route of running
+#
+#   componentize-py -d component.wit -w testworld bindings .
+#
+# to generate the types and protocols to help you write guest code,
+# and then split the code into multiple files, but we're taking a
+# shortcut here so we can write all the guest code in a single file.
+class DemoResourceClass:
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    def greet(self, greeting: str) -> str:
+        return f'{greeting}, {self.name}!'
+
+
+mod = ModuleType("my_interface_name")
+mod.DemoResourceClass = DemoResourceClass
+sys.modules['my_interface_name'] = mod

tests/bindgen/export_resources/app.py

@@ -0,0 +1,13 @@
+package component:basicresource;
+
+interface my-interface-name {
+    interface-func: func(foo: string) -> string;
+    resource demo-resource-class {
+        constructor(name: string);
+        greet: func(greeting: string) -> string;
+    }
+}
+
+world testworld {
+    export my-interface-name;
+}

tests/bindgen/export_resources/component.wit

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+
+def test_bare_funcs(bindgen_testcase):
+    store, root = bindgen_testcase(
+        guest_code_dir=Path(__file__).parent,
+        world_name='testworld',
+    )
+    interface = root.my_interface_name()
+    instance = interface.DemoResourceClass(store, 'myname')
+    result = instance.greet(store, 'Hello there')
+    assert result == 'Hello there, myname!'

tests/bindgen/export_resources/test_export_resources.py

@@ -0,0 +1,15 @@
+from typing import List
+
+
+class Lists:
+    def strings(self, a: str) -> str:
+        return a
+
+    def bytes(self, a: bytes) -> bytes:
+        return a
+
+    def ints(self, a: List[int]) -> List[int]:
+        return a
+
+    def string_list(self, a: List[str]) -> List[str]:
+        return a

tests/bindgen/list_types/app.py

@@ -0,0 +1,8 @@
+package component:lists;
+
+world lists {
+  export strings: func(a: string) -> string;
+  export bytes: func(a: list<u8>) -> list<u8>;
+  export ints: func(a: list<u32>) -> list<u32>;
+  export string-list: func(a: list<string>) -> list<string>;
+}

tests/bindgen/list_types/component.wit

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+
+def test_lists(bindgen_testcase):
+    store, root = bindgen_testcase(
+        guest_code_dir=Path(__file__).parent,
+        world_name='lists',
+    )
+    assert root.strings(store, '') == ''
+    assert root.strings(store, 'a') == 'a'
+    assert root.strings(store, 'hello world') == 'hello world'
+    assert root.strings(store, 'hello ⚑ world') == 'hello ⚑ world'
+
+    assert root.bytes(store, b'') == b''
+    assert root.bytes(store, b'a') == b'a'
+    assert root.bytes(store, b'\x01\x02') == b'\x01\x02'
+
+    assert root.ints(store, []) == []
+    assert root.ints(store, [1]) == [1]
+    assert root.ints(store, [1, 2, 100, 10000]) == [1, 2, 100, 10000]
+
+    assert root.string_list(store, []) == []
+    assert root.string_list(store, ['']) == ['']
+    assert root.string_list(
+        store, ['a', 'b', '', 'd', 'hello']
+    ) == ['a', 'b', '', 'd', 'hello']