esm: add experimental support for addon modules

PR-URL: https://github.com/nodejs/node/pull/55844
Fixes: https://github.com/nodejs/node/issues/40541
Fixes: https://github.com/nodejs/node/issues/55821
Reviewed-By: Guy Bedford <guybedford@gmail.com>
Reviewed-By: Geoffrey Booth <webadmin@geoffreybooth.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: Jacob Smith <jacob@frende.me>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

doc/api/cli.md

@@ -45,8 +45,11 @@ Otherwise, the file is loaded using the CommonJS module loader. See
 
 When loading, the [ES module loader][Modules loaders] loads the program
 entry point, the `node` command will accept as input only files with `.js`,
-`.mjs`, or `.cjs` extensions; and with `.wasm` extensions when
-[`--experimental-wasm-modules`][] is enabled.
+`.mjs`, or `.cjs` extensions. With the following flags, additional file
+extensions are enabled:
+
+* [`--experimental-wasm-modules`][] for files with `.wasm` extension.
+* [`--experimental-addon-modules`][] for files with `.node` extension.
 
 ## Options
 
@@ -880,6 +883,16 @@ and `"` are usable.
 It is possible to run code containing inline types by passing
 [`--experimental-strip-types`][].
 
+### `--experimental-addon-modules`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+Enable experimental import support for `.node` addons.
+
 ### `--experimental-eventsource`
 
 <!-- YAML
@@ -3049,6 +3062,7 @@ one is included in the list below.
 * `--enable-source-maps`
 * `--entry-url`
 * `--experimental-abortcontroller`
+* `--experimental-addon-modules`
 * `--experimental-detect-module`
 * `--experimental-eventsource`
 * `--experimental-import-meta-resolve`
@@ -3620,6 +3634,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [`--diagnostic-dir`]: #--diagnostic-dirdirectory
 [`--env-file-if-exists`]: #--env-file-if-existsconfig
 [`--env-file`]: #--env-fileconfig
+[`--experimental-addon-modules`]: #--experimental-addon-modules
 [`--experimental-sea-config`]: single-executable-applications.md#generating-single-executable-preparation-blobs
 [`--experimental-strip-types`]: #--experimental-strip-types
 [`--experimental-wasm-modules`]: #--experimental-wasm-modules

doc/api/esm.md

@@ -1045,18 +1045,21 @@ _isImports_, _conditions_)
 > 5. If `--experimental-wasm-modules` is enabled and _url_ ends in
 >    _".wasm"_, then
 >    1. Return _"wasm"_.
-> 6. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
-> 7. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
-> 8. Let _packageType_ be **null**.
-> 9. If _pjson?.type_ is _"module"_ or _"commonjs"_, then
->    1. Set _packageType_ to _pjson.type_.
-> 10. If _url_ ends in _".js"_, then
+> 6. If `--experimental-addon-modules` is enabled and _url_ ends in
+>    _".node"_, then
+>    1. Return _"addon"_.
+> 7. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
+> 8. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
+> 9. Let _packageType_ be **null**.
+> 10. If _pjson?.type_ is _"module"_ or _"commonjs"_, then
+>     1. Set _packageType_ to _pjson.type_.
+> 11. If _url_ ends in _".js"_, then
 >     1. If _packageType_ is not **null**, then
 >        1. Return _packageType_.
 >     2. If the result of **DETECT\_MODULE\_SYNTAX**(_source_) is true, then
 >        1. Return _"module"_.
 >     3. Return _"commonjs"_.
-> 11. If _url_ does not have any extension, then
+> 12. If _url_ does not have any extension, then
 >     1. If _packageType_ is _"module"_ and `--experimental-wasm-modules` is
 >        enabled and the file at _url_ contains the header for a WebAssembly
 >        module, then
@@ -1066,7 +1069,7 @@ _isImports_, _conditions_)
 >     3. If the result of **DETECT\_MODULE\_SYNTAX**(_source_) is true, then
 >        1. Return _"module"_.
 >     4. Return _"commonjs"_.
-> 12. Return **undefined** (will throw during load phase).
+> 13. Return **undefined** (will throw during load phase).
 
 **LOOKUP\_PACKAGE\_SCOPE**(_url_)
 

doc/api/module.md

@@ -1161,6 +1161,7 @@ The final value of `format` must be one of the following:
 
 | `format`     | Description                    | Acceptable types for `source` returned by `load`   |
 | ------------ | ------------------------------ | -------------------------------------------------- |
+| `'addon'`    | Load a Node.js addon           | {null}                                             |
 | `'builtin'`  | Load a Node.js builtin module  | {null}                                             |
 | `'commonjs'` | Load a Node.js CommonJS module | {string\|ArrayBuffer\|TypedArray\|null\|undefined} |
 | `'json'`     | Load a JSON file               | {string\|ArrayBuffer\|TypedArray}                  |

doc/node.1

@@ -163,6 +163,9 @@ Enable Source Map V3 support for stack traces.
 .It Fl -entry-url
 Interpret the entry point as a URL.
 .
+.It Fl -experimental-addon-modules
+Enable experimental addon module support.
+.
 .It Fl -experimental-import-meta-resolve
 Enable experimental ES modules support for import.meta.resolve().
 .

lib/internal/modules/esm/formats.js

@@ -10,6 +10,7 @@ const fsBindings = internalBinding('fs');
 const { fs: fsConstants } = internalBinding('constants');
 
 const experimentalWasmModules = getOptionValue('--experimental-wasm-modules');
+const experimentalAddonModules = getOptionValue('--experimental-addon-modules');
 
 const extensionFormatMap = {
   '__proto__': null,
@@ -23,6 +24,10 @@ if (experimentalWasmModules) {
   extensionFormatMap['.wasm'] = 'wasm';
 }
 
+if (experimentalAddonModules) {
+  extensionFormatMap['.node'] = 'addon';
+}
+
 if (getOptionValue('--experimental-strip-types')) {
   extensionFormatMap['.ts'] = 'module-typescript';
   extensionFormatMap['.mts'] = 'module-typescript';

lib/internal/modules/esm/load.js

@@ -105,6 +105,9 @@ async function defaultLoad(url, context = kEmptyObject) {
   if (urlInstance.protocol === 'node:') {
     source = null;
     format ??= 'builtin';
+  } else if (format === 'addon') {
+    // Skip loading addon file content. It must be loaded with dlopen from file system.
+    source = null;
   } else if (format !== 'commonjs') {
     if (source == null) {
       ({ responseURL, source } = await getSource(urlInstance, context));

lib/internal/modules/esm/translators.js

@@ -3,7 +3,6 @@
 const {
   ArrayPrototypeMap,
   ArrayPrototypePush,
-  Boolean,
   FunctionPrototypeCall,
   JSONParse,
   ObjectKeys,
@@ -52,6 +51,7 @@ let debug = require('internal/util/debuglog').debuglog('esm', (fn) => {
 });
 const { emitExperimentalWarning, kEmptyObject, setOwnProperty, isWindows } = require('internal/util');
 const {
+  ERR_INVALID_RETURN_PROPERTY_VALUE,
   ERR_UNKNOWN_BUILTIN_MODULE,
 } = require('internal/errors').codes;
 const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache');
@@ -185,7 +185,7 @@ function createCJSModuleWrap(url, source, isMain, format, loadCJS = loadCJSModul
   // In case the source was not provided by the `load` step, we need fetch it now.
   source = stringify(source ?? getSource(new URL(url)).source);
 
-  const { exportNames, module } = cjsPreparseModuleExports(filename, source, isMain, format);
+  const { exportNames, module } = cjsPreparseModuleExports(filename, source, format);
   cjsCache.set(url, module);
 
   const wrapperNames = [...exportNames, 'module.exports'];
@@ -229,6 +229,47 @@ function createCJSModuleWrap(url, source, isMain, format, loadCJS = loadCJSModul
   }, module);
 }
 
+/**
+ * Creates a ModuleWrap object for a CommonJS module without source texts.
+ * @param {string} url - The URL of the module.
+ * @param {boolean} isMain - Whether the module is the main module.
+ * @returns {ModuleWrap} The ModuleWrap object for the CommonJS module.
+ */
+function createCJSNoSourceModuleWrap(url, isMain) {
+  debug(`Translating CJSModule without source ${url}`);
+
+  const filename = urlToFilename(url);
+
+  const module = cjsEmplaceModuleCacheEntry(filename);
+  cjsCache.set(url, module);
+
+  if (isMain) {
+    setOwnProperty(process, 'mainModule', module);
+  }
+
+  // Addon export names are not known until the addon is loaded.
+  const exportNames = ['default', 'module.exports'];
+  return new ModuleWrap(url, undefined, exportNames, function evaluationCallback() {
+    debug(`Loading CJSModule ${url}`);
+
+    if (!module.loaded) {
+      wrapModuleLoad(filename, null, isMain);
+    }
+
+    /** @type {import('./loader').ModuleExports} */
+    let exports;
+    if (module[kModuleExport] !== undefined) {
+      exports = module[kModuleExport];
+      module[kModuleExport] = undefined;
+    } else {
+      ({ exports } = module);
+    }
+
+    this.setExport('default', exports);
+    this.setExport('module.exports', exports);
+  }, module);
+}
+
 translators.set('commonjs-sync', function requireCommonJS(url, source, isMain) {
   initCJSParseSync();
 
@@ -280,26 +321,38 @@ translators.set('commonjs', function commonjsStrategy(url, source, isMain) {
   return createCJSModuleWrap(url, source, isMain, 'commonjs', cjsLoader);
 });
 
+/**
+ * Get or create an entry in the CJS module cache for the given filename.
+ * @param {string} filename CJS module filename
+ * @returns {CJSModule} the cached CJS module entry
+ */
+function cjsEmplaceModuleCacheEntry(filename, exportNames) {
+  // TODO: Do we want to keep hitting the user mutable CJS loader here?
+  let cjsMod = CJSModule._cache[filename];
+  if (cjsMod) {
+    return cjsMod;
+  }
+
+  cjsMod = new CJSModule(filename);
+  cjsMod.filename = filename;
+  cjsMod.paths = CJSModule._nodeModulePaths(cjsMod.path);
+  cjsMod[kIsCachedByESMLoader] = true;
+  CJSModule._cache[filename] = cjsMod;
+
+  return cjsMod;
+}
+
 /**
  * Pre-parses a CommonJS module's exports and re-exports.
  * @param {string} filename - The filename of the module.
  * @param {string} [source] - The source code of the module.
- * @param {boolean} isMain - Whether it is pre-parsing for the entry point.
- * @param {string} format
+ * @param {string} [format]
  */
-function cjsPreparseModuleExports(filename, source, isMain, format) {
-  let module = CJSModule._cache[filename];
-  if (module && module[kModuleExportNames] !== undefined) {
+function cjsPreparseModuleExports(filename, source, format) {
+  const module = cjsEmplaceModuleCacheEntry(filename);
+  if (module[kModuleExportNames] !== undefined) {
     return { module, exportNames: module[kModuleExportNames] };
   }
-  const loaded = Boolean(module);
-  if (!loaded) {
-    module = new CJSModule(filename);
-    module.filename = filename;
-    module.paths = CJSModule._nodeModulePaths(module.path);
-    module[kIsCachedByESMLoader] = true;
-    CJSModule._cache[filename] = module;
-  }
 
   if (source === undefined) {
     ({ source } = loadSourceForCJSWithHooks(module, filename, format));
@@ -340,7 +393,7 @@ function cjsPreparseModuleExports(filename, source, isMain, format) {
 
       if (format === 'commonjs' ||
         (!BuiltinModule.normalizeRequirableId(resolved) && findLongestRegisteredExtension(resolved) === '.js')) {
-        const { exportNames: reexportNames } = cjsPreparseModuleExports(resolved, undefined, false, format);
+        const { exportNames: reexportNames } = cjsPreparseModuleExports(resolved, undefined, format);
         for (const name of reexportNames) {
           exportNames.add(name);
         }
@@ -462,6 +515,25 @@ translators.set('wasm', async function(url, source) {
   }).module;
 });
 
+// Strategy for loading a addon
+translators.set('addon', function translateAddon(url, source, isMain) {
+  emitExperimentalWarning('Importing addons');
+
+  // The addon must be loaded from file system with dlopen. Assert
+  // the source is null.
+  if (source !== null) {
+    throw new ERR_INVALID_RETURN_PROPERTY_VALUE(
+      'null',
+      'load',
+      'source',
+      source);
+  }
+
+  debug(`Translating addon ${url}`);
+
+  return createCJSNoSourceModuleWrap(url, isMain);
+});
+
 // Strategy for loading a commonjs TypeScript module
 translators.set('commonjs-typescript', function(url, source) {
   emitExperimentalWarning('Type Stripping');

src/node_options.cc

@@ -409,6 +409,10 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             "Treat the entrypoint as a URL",
             &EnvironmentOptions::entry_is_url,
             kAllowedInEnvvar);
+  AddOption("--experimental-addon-modules",
+            "experimental import support for addons",
+            &EnvironmentOptions::experimental_addon_modules,
+            kAllowedInEnvvar);
   AddOption("--experimental-abortcontroller", "", NoOp{}, kAllowedInEnvvar);
   AddOption("--experimental-eventsource",
             "experimental EventSource API",

src/node_options.h

@@ -120,6 +120,7 @@ class EnvironmentOptions : public Options {
   bool require_module = true;
   std::string dns_result_order;
   bool enable_source_maps = false;
+  bool experimental_addon_modules = false;
   bool experimental_eventsource = false;
   bool experimental_fetch = true;
   bool experimental_websocket = true;

test/addons/.gitignore

@@ -5,3 +5,4 @@ Makefile
 *.mk
 gyp-mac-tool
 /*/build
+/esm/node_modules/*/build

test/addons/esm/binding-export-default.cc

@@ -0,0 +1,17 @@
+#include <node.h>
+#include <uv.h>
+#include <v8.h>
+
+static void Method(const v8::FunctionCallbackInfo<v8::Value>& args) {
+  v8::Isolate* isolate = args.GetIsolate();
+  args.GetReturnValue().Set(
+      v8::String::NewFromUtf8(isolate, "hello world").ToLocalChecked());
+}
+
+static void InitModule(v8::Local<v8::Object> exports,
+                       v8::Local<v8::Value> module,
+                       v8::Local<v8::Context> context) {
+  NODE_SET_METHOD(exports, "default", Method);
+}
+
+NODE_MODULE_CONTEXT_AWARE(Binding, InitModule)

test/addons/esm/binding-export-primitive.cc

@@ -0,0 +1,17 @@
+#include <node.h>
+#include <uv.h>
+#include <v8.h>
+
+static void InitModule(v8::Local<v8::Object> exports,
+                       v8::Local<v8::Value> module_val,
+                       v8::Local<v8::Context> context) {
+  v8::Isolate* isolate = context->GetIsolate();
+  v8::Local<v8::Object> module = module_val.As<v8::Object>();
+  module
+      ->Set(context,
+            v8::String::NewFromUtf8(isolate, "exports").ToLocalChecked(),
+            v8::String::NewFromUtf8(isolate, "hello world").ToLocalChecked())
+      .FromJust();
+}
+
+NODE_MODULE_CONTEXT_AWARE(Binding, InitModule)

test/addons/esm/binding.cc

@@ -0,0 +1,17 @@
+#include <node.h>
+#include <uv.h>
+#include <v8.h>
+
+static void Method(const v8::FunctionCallbackInfo<v8::Value>& args) {
+  v8::Isolate* isolate = args.GetIsolate();
+  args.GetReturnValue().Set(
+      v8::String::NewFromUtf8(isolate, "world").ToLocalChecked());
+}
+
+static void InitModule(v8::Local<v8::Object> exports,
+                       v8::Local<v8::Value> module,
+                       v8::Local<v8::Context> context) {
+  NODE_SET_METHOD(exports, "hello", Method);
+}
+
+NODE_MODULE_CONTEXT_AWARE(Binding, InitModule)

test/addons/esm/binding.gyp

@@ -0,0 +1,40 @@
+{
+  'variables': {
+    'source_dir': '<!(<(python) -c "import os; print(os.getcwd())")',
+  },
+  'targets': [
+    {
+      'target_name': 'binding',
+      'sources': [ 'binding.cc' ],
+      'includes': ['../common.gypi'],
+    },
+    {
+      'target_name': 'binding-export-default',
+      'sources': [ 'binding-export-default.cc' ],
+      'includes': ['../common.gypi'],
+    },
+    {
+      'target_name': 'binding-export-primitive',
+      'sources': [ 'binding-export-primitive.cc' ],
+      'includes': ['../common.gypi'],
+    },
+    {
+      'target_name': 'node_modules',
+      'type': 'none',
+      'dependencies': [
+        'binding',
+      ],
+      # The `exports` in package.json can not reference files outside the package
+      # directory. Copy the `binding.node` into the package directory so that
+      # it can be exported with the conditional exports.
+      'copies': [
+        {
+          'files': [
+            '<(PRODUCT_DIR)/binding.node'
+          ],
+          'destination': '<(source_dir)/node_modules/esm-package/build',
+        },
+      ],
+    }
+  ]
+}

test/addons/esm/node_modules/esm-package/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "esm-package",
+  "exports": {
+    "./binding": {
+      "node-addons": "./build/binding.node"
+    }
+  }
+}

test/addons/esm/test-esm.mjs

@@ -0,0 +1,58 @@
+/**
+ * This file is supposed to be loaded by `test-import.js` and `test-require.js`
+ * to verify that `import('*.node')` is working properly either been loaded with
+ * the ESM loader or the CJS loader.
+ */
+
+// eslint-disable-next-line node-core/require-common-first
+import { buildType } from '../../common/index.mjs';
+import assert from 'node:assert';
+import { createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+
+const require = createRequire(import.meta.url);
+
+export async function run() {
+  // binding.node
+  {
+    const bindingPath = require.resolve(`./build/${buildType}/binding.node`);
+    // Test with order of import+require
+    const { default: binding, 'module.exports': exports } = await import(pathToFileURL(bindingPath));
+    assert.strictEqual(binding, exports);
+    assert.strictEqual(binding.hello(), 'world');
+
+    const bindingRequire = require(bindingPath);
+    assert.strictEqual(binding, bindingRequire);
+    assert.strictEqual(binding.hello, bindingRequire.hello);
+
+    // Test multiple loading of the same addon.
+    // ESM cache can not be removed.
+    delete require.cache[bindingPath];
+    const { default: rebinding } = await import(pathToFileURL(bindingPath));
+    assert.strictEqual(rebinding.hello(), 'world');
+    assert.strictEqual(binding.hello, rebinding.hello);
+  }
+
+  // binding-export-default.node
+  {
+    const bindingPath = require.resolve(`./build/${buildType}/binding-export-default.node`);
+    // Test with order of require+import
+    const bindingRequire = require(bindingPath);
+    const ns = await import(pathToFileURL(bindingPath));
+    assert.strictEqual(ns.default, bindingRequire);
+
+    // As same as ESM-import-CJS, the default export is the value of `module.exports`.
+    assert.strictEqual(ns.default, ns['module.exports']);
+    assert.strictEqual(ns.default.default(), 'hello world');
+  }
+
+  // binding-export-primitive.node
+  {
+    const bindingPath = require.resolve(`./build/${buildType}/binding-export-primitive.node`);
+    const ns = await import(pathToFileURL(bindingPath));
+
+    // As same as ESM-import-CJS, the default export is the value of `module.exports`.
+    assert.strictEqual(ns.default, ns['module.exports']);
+    assert.strictEqual(ns.default, 'hello world');
+  }
+}

test/addons/esm/test-import-package.js

@@ -0,0 +1,15 @@
+// Flags: --experimental-addon-modules
+'use strict';
+const common = require('../../common');
+const assert = require('node:assert');
+
+/**
+ * Test that the export condition `node-addons` can be used
+ * with `*.node` files with the ESM loader.
+ */
+
+import('esm-package/binding')
+  .then((mod) => {
+    assert.strictEqual(mod.default.hello(), 'world');
+  })
+  .then(common.mustCall());

test/addons/esm/test-import.js

@@ -0,0 +1,7 @@
+// Flags: --experimental-addon-modules
+'use strict';
+const common = require('../../common');
+
+import('./test-esm.mjs')
+  .then((mod) => mod.run())
+  .then(common.mustCall());

test/addons/esm/test-require-package.js

@@ -0,0 +1,12 @@
+// Flags: --experimental-addon-modules
+'use strict';
+require('../../common');
+const assert = require('node:assert');
+
+/**
+ * Test that the export condition `node-addons` can be used
+ * with `*.node` files with the CJS loader.
+ */
+
+const mod = require('esm-package/binding');
+assert.strictEqual(mod.hello(), 'world');

test/addons/esm/test-require.js

@@ -0,0 +1,6 @@
+// Flags: --experimental-addon-modules
+'use strict';
+const common = require('../../common');
+
+require('./test-esm.mjs')
+  .run().then(common.mustCall());