feat: support rebuild and build for cross-compiling Node-API module to wasm on Windows (#2974)

.github/workflows/tests.yml

@@ -135,7 +135,7 @@ jobs:
           FULL_TEST: ${{ (matrix.node == '20.x' && matrix.python == '3.12') && '1' || '0' }}
       - name: Run Tests (Windows)
         if: startsWith(matrix.os, 'windows')
-        shell: pwsh
-        run: npm run test --python="${env:pythonLocation}\\python.exe"
+        shell: bash # Building wasm on Windows requires using make generator, it only works in bash
+        run: npm run test --python="${pythonLocation}\\python.exe"
         env:
           FULL_TEST: ${{ (matrix.node == '20.x' && matrix.python == '3.12') && '1' || '0' }}

lib/build.js

@@ -1,6 +1,7 @@
 'use strict'
 
-const fs = require('graceful-fs').promises
+const gracefulFs = require('graceful-fs')
+const fs = gracefulFs.promises
 const path = require('path')
 const { glob } = require('glob')
 const log = require('./log')
@@ -85,59 +86,65 @@ async function build (gyp, argv) {
   async function findSolutionFile () {
     const files = await glob('build/*.sln')
     if (files.length === 0) {
-      throw new Error('Could not find *.sln file. Did you run "configure"?')
+      if (gracefulFs.existsSync('build/Makefile') || (await glob('build/*.mk')).length !== 0) {
+        command = makeCommand
+        await doWhich(false)
+        return
+      } else {
+        throw new Error('Could not find *.sln file or Makefile. Did you run "configure"?')
+      }
     }
     guessedSolution = files[0]
     log.verbose('found first Solution file', guessedSolution)
-    await doWhich()
+    await doWhich(true)
   }
 
   /**
    * Uses node-which to locate the msbuild / make executable.
    */
 
-  async function doWhich () {
+  async function doWhich (msvs) {
     // On Windows use msbuild provided by node-gyp configure
-    if (win) {
+    if (msvs) {
       if (!config.variables.msbuild_path) {
         throw new Error('MSBuild is not set, please run `node-gyp configure`.')
       }
       command = config.variables.msbuild_path
       log.verbose('using MSBuild:', command)
-      await doBuild()
+      await doBuild(msvs)
       return
     }
 
     // First make sure we have the build command in the PATH
     const execPath = await which(command)
     log.verbose('`which` succeeded for `' + command + '`', execPath)
-    await doBuild()
+    await doBuild(msvs)
   }
 
   /**
    * Actually spawn the process and compile the module.
    */
 
-  async function doBuild () {
+  async function doBuild (msvs) {
     // Enable Verbose build
     const verbose = log.logger.isVisible('verbose')
     let j
 
-    if (!win && verbose) {
+    if (!msvs && verbose) {
       argv.push('V=1')
     }
 
-    if (win && !verbose) {
+    if (msvs && !verbose) {
       argv.push('/clp:Verbosity=minimal')
     }
 
-    if (win) {
+    if (msvs) {
       // Turn off the Microsoft logo on Windows
       argv.push('/nologo')
     }
 
     // Specify the build type, Release by default
-    if (win) {
+    if (msvs) {
       // Convert .gypi config target_arch to MSBuild /Platform
       // Since there are many ways to state '32-bit Intel', default to it.
       // N.B. msbuild's Condition string equality tests are case-insensitive.
@@ -173,7 +180,7 @@ async function build (gyp, argv) {
       }
     }
 
-    if (win) {
+    if (msvs) {
       // did the user specify their own .sln file?
       const hasSln = argv.some(function (arg) {
         return path.extname(arg) === '.sln'

lib/configure.js

@@ -92,8 +92,28 @@ async function configure (gyp, argv) {
     log.verbose(
       'build dir', '"build" dir needed to be created?', isNew ? 'Yes' : 'No'
     )
-    const vsInfo = win ? await findVisualStudio(release.semver, gyp.opts['msvs-version']) : null
-    return createConfigFile(vsInfo)
+    if (win) {
+      let usingMakeGenerator = false
+      for (let i = argv.length - 1; i >= 0; --i) {
+        const arg = argv[i]
+        if (arg === '-f' || arg === '--format') {
+          const format = argv[i + 1]
+          if (typeof format === 'string' && format.startsWith('make')) {
+            usingMakeGenerator = true
+            break
+          }
+        } else if (arg.startsWith('--format=make')) {
+          usingMakeGenerator = true
+          break
+        }
+      }
+      let vsInfo = {}
+      if (!usingMakeGenerator) {
+        vsInfo = await findVisualStudio(release.semver, gyp.opts['msvs-version'])
+      }
+      return createConfigFile(vsInfo)
+    }
+    return createConfigFile(null)
   }
 
   async function createConfigFile (vsInfo) {

test/node_modules/hello_napi/binding.gyp

@@ -0,0 +1,8 @@
+{
+  "targets": [
+    {
+      "target_name": "hello",
+      "sources": [ "hello.c" ],
+    }
+  ]
+}

test/node_modules/hello_napi/common.gypi

@@ -0,0 +1,110 @@
+{
+  'variables': {
+    # OS: 'emscripten' | 'wasi' | 'unknown' | 'wasm'
+    'clang': 1,
+    'target_arch%': 'wasm32',
+    'stack_size%': 1048576,
+    'initial_memory%': 16777216,
+    'max_memory%': 2147483648,
+  },
+
+  'target_defaults': {
+    'type': 'executable',
+
+    'defines': [
+      'BUILDING_NODE_EXTENSION',
+      '__STDC_FORMAT_MACROS',
+    ],
+
+    'cflags': [
+      '-Wall',
+      '-Wextra',
+      '-Wno-unused-parameter',
+      '--target=wasm32-unknown-unknown',
+    ],
+    'cflags_cc': [
+      '-fno-rtti',
+      '-fno-exceptions',
+      '-std=c++17'
+    ],
+    'ldflags': [
+      '--target=wasm32-unknown-unknown',
+    ],
+
+    'xcode_settings': {
+      # WARNING_CFLAGS == cflags
+      # OTHER_CFLAGS == cflags_c
+      # OTHER_CPLUSPLUSFLAGS == cflags_cc
+      # OTHER_LDFLAGS == ldflags
+
+      'CLANG_CXX_LANGUAGE_STANDARD': 'c++17',
+      'GCC_ENABLE_CPP_RTTI': 'NO',
+      'GCC_ENABLE_CPP_EXCEPTIONS': 'NO',
+      'WARNING_CFLAGS': [
+        '-Wall',
+        '-Wextra',
+        '-Wno-unused-parameter',
+        '--target=wasm32-unknown-unknown'
+      ],
+      'OTHER_LDFLAGS': [ '--target=wasm32-unknown-unknown' ],
+    },
+
+    'default_configuration': 'Release',
+    'configurations': {
+      'Debug': {
+        'defines': [ 'DEBUG', '_DEBUG' ],
+        'cflags': [ '-g', '-O0' ],
+        'ldflags': [ '-g', '-O0' ],
+        'xcode_settings': {
+          'WARNING_CFLAGS': [ '-g', '-O0' ],
+          'OTHER_LDFLAGS': [ '-g', '-O0' ],
+        },
+      },
+      'Release': {
+        'cflags': [ '-O3' ],
+        'ldflags': [ '-O3', '-Wl,--strip-debug' ],
+        'xcode_settings': {
+          'WARNING_CFLAGS': [ '-O3' ],
+          'OTHER_LDFLAGS': [ '-O3', '-Wl,--strip-debug' ],
+        },
+      }
+    },
+
+    'target_conditions': [
+      ['_type=="executable"', {
+
+        'product_extension': 'wasm',
+
+        'ldflags': [
+          '-Wl,--export-dynamic',
+          '-Wl,--export=napi_register_wasm_v1',
+          '-Wl,--export-if-defined=node_api_module_get_api_version_v1',
+          '-Wl,--import-undefined',
+          '-Wl,--export-table',
+          '-Wl,-zstack-size=<(stack_size)',
+          '-Wl,--initial-memory=<(initial_memory)',
+          '-Wl,--max-memory=<(max_memory)',
+          '-nostdlib',
+          '-Wl,--no-entry',
+        ],
+        'xcode_settings': {
+          'OTHER_LDFLAGS': [
+            '-Wl,--export-dynamic',
+            '-Wl,--export=napi_register_wasm_v1',
+            '-Wl,--export-if-defined=node_api_module_get_api_version_v1',
+            '-Wl,--import-undefined',
+            '-Wl,--export-table',
+            '-Wl,-zstack-size=<(stack_size)',
+            '-Wl,--initial-memory=<(initial_memory)',
+            '-Wl,--max-memory=<(max_memory)',
+            '-nostdlib',
+            '-Wl,--no-entry',
+          ],
+        },
+        'defines': [
+          'PAGESIZE=65536'
+        ],
+      }],
+    ],
+  }
+}

test/node_modules/hello_napi/hello.c

@@ -0,0 +1,54 @@
+#include <stddef.h>
+
+#if !defined(__wasm__) || (defined(__EMSCRIPTEN__) || defined(__wasi__))
+#include <assert.h>
+#include <node_api.h>
+#else
+#define assert(x) do { if (!(x)) { __builtin_trap(); } } while (0)
+
+
+__attribute__((__import_module__("napi")))
+int napi_create_string_utf8(void* env,
+                            const char* str,
+                            size_t length,
+                            void** result);
+
+__attribute__((__import_module__("napi")))
+int napi_create_function(void* env,
+                        const char* utf8name,
+                        size_t length,
+                        void* cb,
+                        void* data,
+                        void** result);
+
+__attribute__((__import_module__("napi")))
+int napi_set_named_property(void* env,
+                            void* object,
+                            const char* utf8name,
+                            void* value);
+#ifdef __cplusplus
+#define EXTERN_C extern "C" {
+#else
+#define EXTERN_C
+#endif
+#define NAPI_MODULE_INIT() \
+  EXTERN_C __attribute__((visibility("default"))) void* napi_register_wasm_v1(void* env, void* exports)
+
+typedef void* napi_env;
+typedef void* napi_value;
+typedef void* napi_callback_info;
+#endif
+
+static napi_value hello(napi_env env, napi_callback_info info) {
+  napi_value greeting = NULL;
+  assert(0 == napi_create_string_utf8(env, "world", -1, &greeting));
+  return greeting;
+}
+
+NAPI_MODULE_INIT() {
+  napi_value hello_function = NULL;
+  assert(0 == napi_create_function(env, "hello", -1,
+                                         hello, NULL, &hello_function));
+  assert(0 == napi_set_named_property(env, exports, "hello", hello_function));
+  return exports;
+}

test/node_modules/hello_napi/hello.js

@@ -0,0 +1,57 @@
+const path = require('path')
+const fs = require('fs')
+
+const addon = (function () {
+  const entry = (() => {
+    try {
+      return require.resolve('./build/Release/hello.node')
+    } catch (_) {
+      return require.resolve('./build/Release/hello.wasm')
+    }
+  })()
+  
+  const ext = path.extname(entry)
+  if (ext === '.node') {
+    return require(entry)
+  }
+
+  if (ext === '.wasm') {
+    const values = [undefined, undefined, null, false, true, global, {}]
+    const module = new WebAssembly.Module(fs.readFileSync(entry))
+    const instance = new WebAssembly.Instance(module, {
+      napi: {
+        napi_create_string_utf8: (env, str, len, ret) => {
+          let end = str
+          const buffer = new Uint8Array(instance.exports.memory.buffer)
+          while (buffer[end]) end++
+          values.push(new TextDecoder().decode(buffer.slice(str, end)))
+          new DataView(instance.exports.memory.buffer).setInt32(ret, values.length - 1, true)
+          return 0
+        },
+        napi_create_function: (env, name, len, fn, data, ret) => {
+          values.push(function () {
+            return values[instance.exports.__indirect_function_table.get(fn)(env, 0)]
+          })
+          new DataView(instance.exports.memory.buffer).setInt32(ret, values.length - 1, true)
+          return 0
+        },
+        napi_set_named_property: (env, obj, key, val) => {
+          const buffer = new Uint8Array(instance.exports.memory.buffer)
+          let end = key
+          while (buffer[end]) end++
+          const k = new TextDecoder().decode(buffer.slice(key, end))
+          values[obj][k] = values[val]
+          return 0
+        }
+      }
+    })
+    const newExports = values[instance.exports.napi_register_wasm_v1(1, 6)]
+    if (newExports) {
+      values[6] = newExports
+    }
+    return values[6]
+  }
+  throw new Error('Failed to initialize Node-API wasm module')
+})()
+
+exports.hello = function() { return addon.hello() }

test/node_modules/hello_napi/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "hello_napi",
+  "version": "0.0.0",
+  "description": "Node.js Addons Example #2",
+  "main": "hello.js",
+  "private": true,
+  "scripts": {
+    "test": "node hello.js"
+  },
+  "gypfile": true
+}

test/test-windows-make.js

@@ -0,0 +1,108 @@
+'use strict'
+
+const { describe, it } = require('mocha')
+const assert = require('assert')
+const path = require('path')
+const gracefulFs = require('graceful-fs')
+const cp = require('child_process')
+const util = require('../lib/util')
+const { platformTimeout } = require('./common')
+
+const addonPath = path.resolve(__dirname, 'node_modules', 'hello_napi')
+const nodeGyp = path.resolve(__dirname, '..', 'bin', 'node-gyp.js')
+
+const execFileSync = (...args) => cp.execFileSync(...args).toString().trim()
+
+const execFile = async (cmd, env) => {
+  const [err,, stderr] = await util.execFile(process.execPath, cmd, {
+    env: {
+      ...process.env,
+      NODE_GYP_NULL_LOGGER: undefined,
+      ...env
+    },
+    encoding: 'utf-8'
+  })
+  return [err, stderr.toString().trim().split(/\r?\n/)]
+}
+
+function runHello (hostProcess = process.execPath) {
+  const testCode = "console.log(require('hello_napi').hello())"
+  return execFileSync(hostProcess, ['--experimental-wasi-unstable-preview1', '-e', testCode], { cwd: __dirname })
+}
+
+function executable (name) {
+  return name + (process.platform === 'win32' ? '.exe' : '')
+}
+
+function getEnv (target) {
+  const env = {
+    GYP_CROSSCOMPILE: '1',
+    AR_host: 'ar',
+    CC_host: 'clang',
+    CXX_host: 'clang++'
+  }
+  if (target === 'emscripten') {
+    env.AR_target = 'emar'
+    env.CC_target = 'emcc'
+    env.CXX_target = 'em++'
+  } else if (target === 'wasi') {
+    env.AR_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('ar'))
+    env.CC_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang'))
+    env.CXX_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang++'))
+  } else if (target === 'wasm') {
+    env.AR_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('ar'))
+    env.CC_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang'))
+    env.CXX_target = path.resolve(__dirname, '..', process.env.WASI_SDK_PATH, 'bin', executable('clang++'))
+    env.CFLAGS = '--target=wasm32'
+  } else if (target === 'win-clang') {
+    let vsdir = 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise'
+    if (!gracefulFs.existsSync(vsdir)) {
+      vsdir = 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community'
+    }
+    const llvmBin = 'VC\\Tools\\Llvm\\x64\\bin'
+    env.AR_target = path.join(vsdir, llvmBin, 'llvm-ar.exe')
+    env.CC_target = path.join(vsdir, llvmBin, 'clang.exe')
+    env.CXX_target = path.join(vsdir, llvmBin, 'clang++.exe')
+    env.CFLAGS = '--target=wasm32'
+  }
+  return env
+}
+
+function quote (path) {
+  if (path.includes(' ')) {
+    return `"${path}"`
+  }
+}
+
+describe('windows-cross-compile', function () {
+  it('build simple node-api addon', async function () {
+    if (process.platform !== 'win32') {
+      return this.skip('This test is only for windows')
+    }
+    const env = getEnv('win-clang')
+    if (!gracefulFs.existsSync(env.CC_target)) {
+      return this.skip('Visual Studio Clang is not installed')
+    }
+
+    // handle bash whitespace
+    env.AR_target = quote(env.AR_target)
+    env.CC_target = quote(env.CC_target)
+    env.CXX_target = quote(env.CXX_target)
+    this.timeout(platformTimeout(1, { win32: 5 }))
+
+    const cmd = [
+      nodeGyp,
+      'rebuild',
+      '-C', addonPath,
+      '--loglevel=verbose',
+      `--nodedir=${addonPath}`,
+      '--arch=wasm32',
+      '--', '-f', 'make'
+    ]
+    const [err, logLines] = await execFile(cmd, env)
+    const lastLine = logLines[logLines.length - 1]
+    assert.strictEqual(err, null)
+    assert.strictEqual(lastLine, 'gyp info ok', 'should end in ok')
+    assert.strictEqual(runHello(), 'world')
+  })
+})