From 6318d2b210224415ff5932c2863e6cc14d4583dc Mon Sep 17 00:00:00 2001 From: Toyo Li Date: Fri, 28 Jun 2024 22:36:30 +0800 Subject: [PATCH] feat: support `rebuild` and `build` for cross-compiling Node-API module to wasm on Windows (#2974) --- .github/workflows/tests.yml | 4 +- lib/build.js | 33 ++++--- lib/configure.js | 24 ++++- test/node_modules/hello_napi/binding.gyp | 8 ++ test/node_modules/hello_napi/common.gypi | 110 ++++++++++++++++++++++ test/node_modules/hello_napi/hello.c | 54 +++++++++++ test/node_modules/hello_napi/hello.js | 57 +++++++++++ test/node_modules/hello_napi/package.json | 11 +++ test/test-windows-make.js | 108 +++++++++++++++++++++ 9 files changed, 392 insertions(+), 17 deletions(-) create mode 100644 test/node_modules/hello_napi/binding.gyp create mode 100644 test/node_modules/hello_napi/common.gypi create mode 100644 test/node_modules/hello_napi/hello.c create mode 100644 test/node_modules/hello_napi/hello.js create mode 100644 test/node_modules/hello_napi/package.json create mode 100644 test/test-windows-make.js diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1e8e8bc8cb..96b08ae3e1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -135,7 +135,7 @@ jobs: FULL_TEST: ${{ (matrix.node == '20.x' && matrix.python == '3.12') && '1' || '0' }} - name: Run Tests (Windows) if: startsWith(matrix.os, 'windows') - shell: pwsh - run: npm run test --python="${env:pythonLocation}\\python.exe" + shell: bash # Building wasm on Windows requires using make generator, it only works in bash + run: npm run test --python="${pythonLocation}\\python.exe" env: FULL_TEST: ${{ (matrix.node == '20.x' && matrix.python == '3.12') && '1' || '0' }} diff --git a/lib/build.js b/lib/build.js index 6b8d84d3ed..e1f49bb6ff 100644 --- a/lib/build.js +++ b/lib/build.js @@ -1,6 +1,7 @@ 'use strict' -const fs = require('graceful-fs').promises +const gracefulFs = require('graceful-fs') +const fs = gracefulFs.promises const path = require('path') const { glob } = require('glob') const log = require('./log') @@ -85,59 +86,65 @@ async function build (gyp, argv) { async function findSolutionFile () { const files = await glob('build/*.sln') if (files.length === 0) { - throw new Error('Could not find *.sln file. Did you run "configure"?') + if (gracefulFs.existsSync('build/Makefile') || (await glob('build/*.mk')).length !== 0) { + command = makeCommand + await doWhich(false) + return + } else { + throw new Error('Could not find *.sln file or Makefile. Did you run "configure"?') + } } guessedSolution = files[0] log.verbose('found first Solution file', guessedSolution) - await doWhich() + await doWhich(true) } /** * Uses node-which to locate the msbuild / make executable. */ - async function doWhich () { + async function doWhich (msvs) { // On Windows use msbuild provided by node-gyp configure - if (win) { + if (msvs) { if (!config.variables.msbuild_path) { throw new Error('MSBuild is not set, please run `node-gyp configure`.') } command = config.variables.msbuild_path log.verbose('using MSBuild:', command) - await doBuild() + await doBuild(msvs) return } // First make sure we have the build command in the PATH const execPath = await which(command) log.verbose('`which` succeeded for `' + command + '`', execPath) - await doBuild() + await doBuild(msvs) } /** * Actually spawn the process and compile the module. */ - async function doBuild () { + async function doBuild (msvs) { // Enable Verbose build const verbose = log.logger.isVisible('verbose') let j - if (!win && verbose) { + if (!msvs && verbose) { argv.push('V=1') } - if (win && !verbose) { + if (msvs && !verbose) { argv.push('/clp:Verbosity=minimal') } - if (win) { + if (msvs) { // Turn off the Microsoft logo on Windows argv.push('/nologo') } // Specify the build type, Release by default - if (win) { + if (msvs) { // Convert .gypi config target_arch to MSBuild /Platform // Since there are many ways to state '32-bit Intel', default to it. // N.B. msbuild's Condition string equality tests are case-insensitive. @@ -173,7 +180,7 @@ async function build (gyp, argv) { } } - if (win) { + if (msvs) { // did the user specify their own .sln file? const hasSln = argv.some(function (arg) { return path.extname(arg) === '.sln' diff --git a/lib/configure.js b/lib/configure.js index e4b8c94e3d..ee672cfbf2 100644 --- a/lib/configure.js +++ b/lib/configure.js @@ -92,8 +92,28 @@ async function configure (gyp, argv) { log.verbose( 'build dir', '"build" dir needed to be created?', isNew ? 'Yes' : 'No' ) - const vsInfo = win ? await findVisualStudio(release.semver, gyp.opts['msvs-version']) : null - return createConfigFile(vsInfo) + if (win) { + let usingMakeGenerator = false + for (let i = argv.length - 1; i >= 0; --i) { + const arg = argv[i] + if (arg === '-f' || arg === '--format') { + const format = argv[i + 1] + if (typeof format === 'string' && format.startsWith('make')) { + usingMakeGenerator = true + break + } + } else if (arg.startsWith('--format=make')) { + usingMakeGenerator = true + break + } + } + let vsInfo = {} + if (!usingMakeGenerator) { + vsInfo = await findVisualStudio(release.semver, gyp.opts['msvs-version']) + } + return createConfigFile(vsInfo) + } + return createConfigFile(null) } async function createConfigFile (vsInfo) { diff --git a/test/node_modules/hello_napi/binding.gyp b/test/node_modules/hello_napi/binding.gyp new file mode 100644 index 0000000000..cb6a3bdc22 --- /dev/null +++ b/test/node_modules/hello_napi/binding.gyp @@ -0,0 +1,8 @@ +{ + "targets": [ + { + "target_name": "hello", + "sources": [ "hello.c" ], + } + ] +} diff --git a/test/node_modules/hello_napi/common.gypi b/test/node_modules/hello_napi/common.gypi new file mode 100644 index 0000000000..f3c53bbad9 --- /dev/null +++ b/test/node_modules/hello_napi/common.gypi @@ -0,0 +1,110 @@ +{ + 'variables': { + # OS: 'emscripten' | 'wasi' | 'unknown' | 'wasm' + 'clang': 1, + 'target_arch%': 'wasm32', + 'stack_size%': 1048576, + 'initial_memory%': 16777216, + 'max_memory%': 2147483648, + }, + + 'target_defaults': { + 'type': 'executable', + + 'defines': [ + 'BUILDING_NODE_EXTENSION', + '__STDC_FORMAT_MACROS', + ], + + 'cflags': [ + '-Wall', + '-Wextra', + '-Wno-unused-parameter', + '--target=wasm32-unknown-unknown', + ], + 'cflags_cc': [ + '-fno-rtti', + '-fno-exceptions', + '-std=c++17' + ], + 'ldflags': [ + '--target=wasm32-unknown-unknown', + ], + + 'xcode_settings': { + # WARNING_CFLAGS == cflags + # OTHER_CFLAGS == cflags_c + # OTHER_CPLUSPLUSFLAGS == cflags_cc + # OTHER_LDFLAGS == ldflags + + 'CLANG_CXX_LANGUAGE_STANDARD': 'c++17', + 'GCC_ENABLE_CPP_RTTI': 'NO', + 'GCC_ENABLE_CPP_EXCEPTIONS': 'NO', + 'WARNING_CFLAGS': [ + '-Wall', + '-Wextra', + '-Wno-unused-parameter', + '--target=wasm32-unknown-unknown' + ], + 'OTHER_LDFLAGS': [ '--target=wasm32-unknown-unknown' ], + }, + + 'default_configuration': 'Release', + 'configurations': { + 'Debug': { + 'defines': [ 'DEBUG', '_DEBUG' ], + 'cflags': [ '-g', '-O0' ], + 'ldflags': [ '-g', '-O0' ], + 'xcode_settings': { + 'WARNING_CFLAGS': [ '-g', '-O0' ], + 'OTHER_LDFLAGS': [ '-g', '-O0' ], + }, + }, + 'Release': { + 'cflags': [ '-O3' ], + 'ldflags': [ '-O3', '-Wl,--strip-debug' ], + 'xcode_settings': { + 'WARNING_CFLAGS': [ '-O3' ], + 'OTHER_LDFLAGS': [ '-O3', '-Wl,--strip-debug' ], + }, + } + }, + + 'target_conditions': [ + ['_type=="executable"', { + + 'product_extension': 'wasm', + + 'ldflags': [ + '-Wl,--export-dynamic', + '-Wl,--export=napi_register_wasm_v1', + '-Wl,--export-if-defined=node_api_module_get_api_version_v1', + '-Wl,--import-undefined', + '-Wl,--export-table', + '-Wl,-zstack-size=<(stack_size)', + '-Wl,--initial-memory=<(initial_memory)', + '-Wl,--max-memory=<(max_memory)', + '-nostdlib', + '-Wl,--no-entry', + ], + 'xcode_settings': { + 'OTHER_LDFLAGS': [ + '-Wl,--export-dynamic', + '-Wl,--export=napi_register_wasm_v1', + '-Wl,--export-if-defined=node_api_module_get_api_version_v1', + '-Wl,--import-undefined', + '-Wl,--export-table', + '-Wl,-zstack-size=<(stack_size)', + '-Wl,--initial-memory=<(initial_memory)', + '-Wl,--max-memory=<(max_memory)', + '-nostdlib', + '-Wl,--no-entry', + ], + }, + 'defines': [ + 'PAGESIZE=65536' + ], + }], + ], + } +} diff --git a/test/node_modules/hello_napi/hello.c b/test/node_modules/hello_napi/hello.c new file mode 100644 index 0000000000..2c03dc156f --- /dev/null +++ b/test/node_modules/hello_napi/hello.c @@ -0,0 +1,54 @@ +#include + +#if !defined(__wasm__) || (defined(__EMSCRIPTEN__) || defined(__wasi__)) +#include +#include +#else +#define assert(x) do { if (!(x)) { __builtin_trap(); } } while (0) + + +__attribute__((__import_module__("napi"))) +int napi_create_string_utf8(void* env, + const char* str, + size_t length, + void** result); + +__attribute__((__import_module__("napi"))) +int napi_create_function(void* env, + const char* utf8name, + size_t length, + void* cb, + void* data, + void** result); + +__attribute__((__import_module__("napi"))) +int napi_set_named_property(void* env, + void* object, + const char* utf8name, + void* value); +#ifdef __cplusplus +#define EXTERN_C extern "C" { +#else +#define EXTERN_C +#endif +#define NAPI_MODULE_INIT() \ + EXTERN_C __attribute__((visibility("default"))) void* napi_register_wasm_v1(void* env, void* exports) + +typedef void* napi_env; +typedef void* napi_value; +typedef void* napi_callback_info; +#endif + +static napi_value hello(napi_env env, napi_callback_info info) { + napi_value greeting = NULL; + assert(0 == napi_create_string_utf8(env, "world", -1, &greeting)); + return greeting; +} + +NAPI_MODULE_INIT() { + napi_value hello_function = NULL; + assert(0 == napi_create_function(env, "hello", -1, + hello, NULL, &hello_function)); + assert(0 == napi_set_named_property(env, exports, "hello", hello_function)); + return exports; +} diff --git a/test/node_modules/hello_napi/hello.js b/test/node_modules/hello_napi/hello.js new file mode 100644 index 0000000000..00084f851b --- /dev/null +++ b/test/node_modules/hello_napi/hello.js @@ -0,0 +1,57 @@ +const path = require('path') +const fs = require('fs') + +const addon = (function () { + const entry = (() => { + try { + return require.resolve('./build/Release/hello.node') + } catch (_) { + return require.resolve('./build/Release/hello.wasm') + } + })() + + const ext = path.extname(entry) + if (ext === '.node') { + return require(entry) + } + + if (ext === '.wasm') { + const values = [undefined, undefined, null, false, true, global, {}] + const module = new WebAssembly.Module(fs.readFileSync(entry)) + const instance = new WebAssembly.Instance(module, { + napi: { + napi_create_string_utf8: (env, str, len, ret) => { + let end = str + const buffer = new Uint8Array(instance.exports.memory.buffer) + while (buffer[end]) end++ + values.push(new TextDecoder().decode(buffer.slice(str, end))) + new DataView(instance.exports.memory.buffer).setInt32(ret, values.length - 1, true) + return 0 + }, + napi_create_function: (env, name, len, fn, data, ret) => { + values.push(function () { + return values[instance.exports.__indirect_function_table.get(fn)(env, 0)] + }) + new DataView(instance.exports.memory.buffer).setInt32(ret, values.length - 1, true) + return 0 + }, + napi_set_named_property: (env, obj, key, val) => { + const buffer = new Uint8Array(instance.exports.memory.buffer) + let end = key + while (buffer[end]) end++ + const k = new TextDecoder().decode(buffer.slice(key, end)) + values[obj][k] = values[val] + return 0 + } + } + }) + const newExports = values[instance.exports.napi_register_wasm_v1(1, 6)] + if (newExports) { + values[6] = newExports + } + return values[6] + } + throw new Error('Failed to initialize Node-API wasm module') +})() + +exports.hello = function() { return addon.hello() } diff --git a/test/node_modules/hello_napi/package.json b/test/node_modules/hello_napi/package.json new file mode 100644 index 0000000000..6cdb71b21f --- /dev/null +++ b/test/node_modules/hello_napi/package.json @@ -0,0 +1,11 @@ +{ + "name": "hello_napi", + "version": "0.0.0", + "description": "Node.js Addons Example #2", + "main": "hello.js", + "private": true, + "scripts": { + "test": "node hello.js" + }, + "gypfile": true +} diff --git a/test/test-windows-make.js b/test/test-windows-make.js new file mode 100644 index 0000000000..41cefc3035 --- /dev/null +++ b/test/test-windows-make.js @@ -0,0 +1,108 @@ +'use strict' + +const { describe, it } = require('mocha') +const assert = require('assert') +const path = require('path') +const gracefulFs = require('graceful-fs') +const cp = require('child_process') +const util = require('../lib/util') +const { platformTimeout } = require('./common') + +const addonPath = path.resolve(__dirname, 'node_modules', 'hello_napi') +const nodeGyp = path.resolve(__dirname, '..', 'bin', 'node-gyp.js') + +const execFileSync = (...args) => cp.execFileSync(...args).toString().trim() + +const execFile = async (cmd, env) => { + const [err,, stderr] = await util.execFile(process.execPath, cmd, { + env: { + ...process.env, + NODE_GYP_NULL_LOGGER: undefined, + ...env + }, + encoding: 'utf-8' + }) + return [err, stderr.toString().trim().split(/\r?\n/)] +} + +function runHello (hostProcess = process.execPath) { + const testCode = "console.log(require('hello_napi').hello())" + return execFileSync(hostProcess, ['--experimental-wasi-unstable-preview1', '-e', testCode], { cwd: __dirname }) +} + +function executable (name) { + return name + (process.platform === 'win32' ? '.exe' : '') +} + +function getEnv (target) { + const env = { + GYP_CROSSCOMPILE: '1', + AR_host: 'ar', + CC_host: 'clang', + CXX_host: 'clang++' + } + if (target === 'emscripten') { + env.AR_target = 'emar' + env.CC_target = 'emcc' + env.CXX_target = 'em++' + } else if (target === 'wasi') { + env.AR_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('ar')) + env.CC_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang')) + env.CXX_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang++')) + } else if (target === 'wasm') { + env.AR_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('ar')) + env.CC_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang')) + env.CXX_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang++')) + env.CFLAGS = '--target=wasm32' + } else if (target === 'win-clang') { + let vsdir = 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise' + if (!gracefulFs.existsSync(vsdir)) { + vsdir = 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community' + } + const llvmBin = 'VC\\Tools\\Llvm\\x64\\bin' + env.AR_target = path.join(vsdir, llvmBin, 'llvm-ar.exe') + env.CC_target = path.join(vsdir, llvmBin, 'clang.exe') + env.CXX_target = path.join(vsdir, llvmBin, 'clang++.exe') + env.CFLAGS = '--target=wasm32' + } + return env +} + +function quote (path) { + if (path.includes(' ')) { + return `"${path}"` + } +} + +describe('windows-cross-compile', function () { + it('build simple node-api addon', async function () { + if (process.platform !== 'win32') { + return this.skip('This test is only for windows') + } + const env = getEnv('win-clang') + if (!gracefulFs.existsSync(env.CC_target)) { + return this.skip('Visual Studio Clang is not installed') + } + + // handle bash whitespace + env.AR_target = quote(env.AR_target) + env.CC_target = quote(env.CC_target) + env.CXX_target = quote(env.CXX_target) + this.timeout(platformTimeout(1, { win32: 5 })) + + const cmd = [ + nodeGyp, + 'rebuild', + '-C', addonPath, + '--loglevel=verbose', + `--nodedir=${addonPath}`, + '--arch=wasm32', + '--', '-f', 'make' + ] + const [err, logLines] = await execFile(cmd, env) + const lastLine = logLines[logLines.length - 1] + assert.strictEqual(err, null) + assert.strictEqual(lastLine, 'gyp info ok', 'should end in ok') + assert.strictEqual(runHello(), 'world') + }) +})