fs: expose glob and globSync

PR-URL: https://github.com/nodejs/node/pull/51912
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Chemi Atlow <chemi@atlow.co.il>
Reviewed-By: Nitzan Uziely <linkgoron@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

doc/api/fs.md

@@ -1069,6 +1069,38 @@ including subdirectories and files.
 When copying a directory to another directory, globs are not supported and
 behavior is similar to `cp dir1/ dir2/`.
 
+### `fsPromises.glob(pattern[, options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+* `pattern` {string|string\[]}
+* `options` {Object}
+  * `cwd` {string} current working directory. **Default:** `process.cwd()`
+  * `exclude` {Function} Function to filter out files/directories. Return
+    `true` to exclude the item, `false` to include it. **Default:** `undefined`.
+* Returns: {AsyncIterator} An AsyncIterator that yields the paths of files
+  that match the pattern.
+
+```mjs
+import { glob } from 'node:fs/promises';
+
+for await (const entry of glob('**/*.js'))
+  console.log(entry);
+```
+
+```cjs
+const { glob } = require('node:fs/promises');
+
+(async () => {
+  for await (const entry of glob('**/*.js'))
+    console.log(entry);
+})();
+```
+
 ### `fsPromises.lchmod(path, mode)`
 
 <!-- YAML
@@ -3073,6 +3105,44 @@ changes:
 Change the file system timestamps of the object referenced by the supplied file
 descriptor. See [`fs.utimes()`][].
 
+### `fs.glob(pattern[, options], callback)`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+* `pattern` {string|string\[]}
+
+* `options` {Object}
+  * `cwd` {string} current working directory. **Default:** `process.cwd()`
+  * `exclude` {Function} Function to filter out files/directories. Return
+    `true` to exclude the item, `false` to include it. **Default:** `undefined`.
+
+* `callback` {Function}
+  * `err` {Error}
+
+* Retrieves the files matching the specified pattern.
+
+```mjs
+import { glob } from 'node:fs';
+
+glob('**/*.js', (err, matches) => {
+  if (err) throw err;
+  console.log(matches);
+});
+```
+
+```cjs
+const { glob } = require('node:fs');
+
+glob('**/*.js', (err, matches) => {
+  if (err) throw err;
+  console.log(matches);
+});
+```
+
 ### `fs.lchmod(path, mode, callback)`
 
 <!-- YAML
@@ -5529,6 +5599,33 @@ changes:
 
 Synchronous version of [`fs.futimes()`][]. Returns `undefined`.
 
+### `fs.globSync(pattern[, options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+* `pattern` {string|string\[]}
+* `options` {Object}
+  * `cwd` {string} current working directory. **Default:** `process.cwd()`
+  * `exclude` {Function} Function to filter out files/directories. Return
+    `true` to exclude the item, `false` to include it. **Default:** `undefined`.
+* Returns: {string\[]} paths of files that match the pattern.
+
+```mjs
+import { globSync } from 'node:fs';
+
+console.log(globSync('**/*.js'));
+```
+
+```cjs
+const { globSync } = require('node:fs');
+
+console.log(globSync('**/*.js'));
+```
+
 ### `fs.lchmodSync(path, mode)`
 
 <!-- YAML

lib/fs.js

@@ -86,6 +86,8 @@ const {
 const { toPathIfFileURL } = require('internal/url');
 const {
   customPromisifyArgs: kCustomPromisifyArgsSymbol,
+  emitExperimentalWarning,
+  getLazy,
   kEmptyObject,
   promisify: {
     custom: kCustomPromisifiedSymbol,
@@ -3102,6 +3104,38 @@ function createWriteStream(path, options) {
   return new WriteStream(path, options);
 }
 
+const lazyGlob = getLazy(() => require('internal/fs/glob').Glob);
+
+function glob(pattern, options, callback) {
+  emitExperimentalWarning('glob');
+  if (typeof options === 'function') {
+    callback = options;
+    options = undefined;
+  }
+  callback = makeCallback(callback);
+
+  const Glob = lazyGlob();
+  // TODO: Use iterator helpers when available
+  (async () => {
+    try {
+      const res = [];
+      for await (const entry of new Glob(pattern, options).glob()) {
+        ArrayPrototypePush(res, entry);
+      }
+      callback(null, res);
+    } catch (err) {
+      callback(err);
+    }
+  })();
+}
+
+function globSync(pattern, options) {
+  emitExperimentalWarning('globSync');
+  const Glob = lazyGlob();
+  return new Glob(pattern, options).globSync();
+}
+
+
 module.exports = fs = {
   appendFile,
   appendFileSync,
@@ -3135,6 +3169,8 @@ module.exports = fs = {
   ftruncateSync,
   futimes,
   futimesSync,
+  glob,
+  globSync,
   lchown,
   lchownSync,
   lchmod: constants.O_SYMLINK !== undefined ? lchmod : undefined,

lib/internal/fs/glob.js

@@ -1,5 +1,6 @@
 'use strict';
 const { lstatSync, readdirSync } = require('fs');
+const { lstat, readdir } = require('fs/promises');
 const { join, resolve } = require('path');
 
 const {
@@ -8,6 +9,8 @@ const {
 const {
   validateFunction,
   validateObject,
+  validateString,
+  validateStringArray,
 } = require('internal/validators');
 
 const {
@@ -18,6 +21,7 @@ const {
   ArrayPrototypePop,
   ArrayPrototypePush,
   ArrayPrototypeSome,
+  PromisePrototypeThen,
   SafeMap,
   SafeSet,
   StringPrototypeEndsWith,
@@ -37,6 +41,15 @@ class Cache {
   #statsCache = new SafeMap();
   #readdirCache = new SafeMap();
 
+  stat(path) {
+    const cached = this.#statsCache.get(path);
+    if (cached) {
+      return cached;
+    }
+    const promise = PromisePrototypeThen(lstat(path), null, () => null);
+    this.#statsCache.set(path, promise);
+    return promise;
+  }
   statSync(path) {
     const cached = this.#statsCache.get(path);
     if (cached) {
@@ -54,6 +67,15 @@ class Cache {
   addToStatCache(path, val) {
     this.#statsCache.set(path, val);
   }
+  async readdir(path) {
+    const cached = this.#readdirCache.get(path);
+    if (cached) {
+      return cached;
+    }
+    const promise = PromisePrototypeThen(readdir(path, { __proto__: null, withFileTypes: true }), null, () => null);
+    this.#readdirCache.set(path, promise);
+    return promise;
+  }
   readdirSync(path) {
     const cached = this.#readdirCache.get(path);
     if (cached) {
@@ -151,7 +173,8 @@ class Glob {
   #results = new SafeSet();
   #queue = [];
   #subpatterns = new SafeMap();
-  constructor(patterns, options = kEmptyObject) {
+  #patterns;
+  constructor(pattern, options = kEmptyObject) {
     validateObject(options, 'options');
     const { exclude, cwd } = options;
     if (exclude != null) {
@@ -159,6 +182,14 @@ class Glob {
     }
     this.#root = cwd ?? '.';
     this.#exclude = exclude;
+    let patterns;
+    if (typeof pattern === 'object') {
+      validateStringArray(pattern, 'patterns');
+      patterns = pattern;
+    } else {
+      validateString(pattern, 'patterns');
+      patterns = [pattern];
+    }
     this.matchers = ArrayPrototypeMap(patterns, (pattern) => new (lazyMinimatch().Minimatch)(pattern, {
       __proto__: null,
       nocase: isWindows || isOSX,
@@ -169,21 +200,18 @@ class Glob {
       platform: process.platform,
       nocaseMagicOnly: true,
     }));
-  }
 
-  globSync() {
+    this.#patterns = ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
-    ArrayPrototypePush(this.#queue, {
-      __proto__: null,
-      path: '.',
-      patterns: ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
                                                                                          (pattern, i) => new Pattern(
                                                                                            pattern,
                                                                                            matcher.globParts[i],
-                                                                                      new SafeSet([0]),
+                                                                                           new SafeSet().add(0),
                                                                                            new SafeSet(),
-                                                                                    ))),
+                                                                                         )));
-    });
+  }
 
+  globSync() {
+    ArrayPrototypePush(this.#queue, { __proto__: null, path: '.', patterns: this.#patterns });
     while (this.#queue.length > 0) {
       const item = ArrayPrototypePop(this.#queue);
       for (let i = 0; i < item.patterns.length; i++) {
@@ -216,22 +244,22 @@ class Glob {
 
     if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
       // Absolute path, go to root
-      this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet([1])));
+      this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
       return;
     }
     if (isFirst && pattern.at(0) === '') {
       // Absolute path, go to root
-      this.#addSubpattern('/', pattern.child(new SafeSet([1])));
+      this.#addSubpattern('/', pattern.child(new SafeSet().add(1)));
       return;
     }
     if (isFirst && pattern.at(0) === '..') {
       // Start with .., go to parent
-      this.#addSubpattern('../', pattern.child(new SafeSet([1])));
+      this.#addSubpattern('../', pattern.child(new SafeSet().add(1)));
       return;
     }
     if (isFirst && pattern.at(0) === '.') {
       // Start with ., proceed
-      this.#addSubpattern('.', pattern.child(new SafeSet([1])));
+      this.#addSubpattern('.', pattern.child(new SafeSet().add(1)));
       return;
     }
 
@@ -264,7 +292,7 @@ class Glob {
         stat.name = firstPattern;
         children = [stat];
       } else {
-        children = [];
+        return;
       }
     } else {
       children = this.#cache.readdirSync(fullpath);
@@ -329,18 +357,18 @@ class Glob {
             const parent = join(path, '..');
             if (nextIndex < last) {
               if (!this.#subpatterns.has(path) && !this.#cache.seen(path, pattern, nextIndex + 1)) {
-                this.#subpatterns.set(path, [pattern.child(new SafeSet([nextIndex + 1]))]);
+                this.#subpatterns.set(path, [pattern.child(new SafeSet().add(nextIndex + 1))]);
               }
               if (!this.#subpatterns.has(parent) && !this.#cache.seen(parent, pattern, nextIndex + 1)) {
-                this.#subpatterns.set(parent, [pattern.child(new SafeSet([nextIndex + 1]))]);
+                this.#subpatterns.set(parent, [pattern.child(new SafeSet().add(nextIndex + 1))]);
               }
             } else {
               if (!this.#cache.seen(path, pattern, nextIndex)) {
-                this.#cache.add(path, pattern.child(new SafeSet([nextIndex])));
+                this.#cache.add(path, pattern.child(new SafeSet().add(nextIndex)));
                 this.#results.add(path);
               }
               if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) {
-                this.#cache.add(parent, pattern.child(new SafeSet([nextIndex])));
+                this.#cache.add(parent, pattern.child(new SafeSet().add(nextIndex)));
                 this.#results.add(parent);
               }
             }
@@ -376,6 +404,219 @@ class Glob {
       }
     }
   }
+
+
+  async* glob() {
+    ArrayPrototypePush(this.#queue, { __proto__: null, path: '.', patterns: this.#patterns });
+    while (this.#queue.length > 0) {
+      const item = ArrayPrototypePop(this.#queue);
+      for (let i = 0; i < item.patterns.length; i++) {
+        yield* this.#iterateSubpatterns(item.path, item.patterns[i]);
+      }
+      this.#subpatterns
+        .forEach((patterns, path) => ArrayPrototypePush(this.#queue, { __proto__: null, path, patterns }));
+      this.#subpatterns.clear();
+    }
+  }
+  async* #iterateSubpatterns(path, pattern) {
+    const seen = this.#cache.add(path, pattern);
+    if (seen) {
+      return;
+    }
+    const fullpath = resolve(this.#root, path);
+    const stat = await this.#cache.stat(fullpath);
+    const last = pattern.last;
+    const isDirectory = stat?.isDirectory() || (stat?.isSymbolicLink() && pattern.hasSeenSymlinks);
+    const isLast = pattern.isLast(isDirectory);
+    const isFirst = pattern.isFirst();
+
+    if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
+      // Absolute path, go to root
+      this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
+      return;
+    }
+    if (isFirst && pattern.at(0) === '') {
+      // Absolute path, go to root
+      this.#addSubpattern('/', pattern.child(new SafeSet().add(1)));
+      return;
+    }
+    if (isFirst && pattern.at(0) === '..') {
+      // Start with .., go to parent
+      this.#addSubpattern('../', pattern.child(new SafeSet().add(1)));
+      return;
+    }
+    if (isFirst && pattern.at(0) === '.') {
+      // Start with ., proceed
+      this.#addSubpattern('.', pattern.child(new SafeSet().add(1)));
+      return;
+    }
+
+    if (isLast && typeof pattern.at(-1) === 'string') {
+      // Add result if it exists
+      const p = pattern.at(-1);
+      const stat = await this.#cache.stat(join(fullpath, p));
+      if (stat && (p || isDirectory)) {
+        const result = join(path, p);
+        if (!this.#results.has(result)) {
+          this.#results.add(result);
+          yield result;
+        }
+      }
+      if (pattern.indexes.size === 1 && pattern.indexes.has(last)) {
+        return;
+      }
+    } else if (isLast && pattern.at(-1) === lazyMinimatch().GLOBSTAR &&
+     (path !== '.' || pattern.at(0) === '.' || (last === 0 && stat))) {
+      // If pattern ends with **, add to results
+      // if path is ".", add it only if pattern starts with "." or pattern is exactly "**"
+      if (!this.#results.has(path)) {
+        this.#results.add(path);
+        yield path;
+      }
+    }
+
+    if (!isDirectory) {
+      return;
+    }
+
+    let children;
+    const firstPattern = pattern.indexes.size === 1 && pattern.at(pattern.indexes.values().next().value);
+    if (typeof firstPattern === 'string') {
+      const stat = await this.#cache.stat(join(fullpath, firstPattern));
+      if (stat) {
+        stat.name = firstPattern;
+        children = [stat];
+      } else {
+        return;
+      }
+    } else {
+      children = await this.#cache.readdir(fullpath);
+    }
+
+    for (let i = 0; i < children.length; i++) {
+      const entry = children[i];
+      const entryPath = join(path, entry.name);
+      this.#cache.addToStatCache(join(fullpath, entry.name), entry);
+
+      const subPatterns = new SafeSet();
+      const nSymlinks = new SafeSet();
+      for (const index of pattern.indexes) {
+        // For each child, chek potential patterns
+        if (this.#cache.seen(entryPath, pattern, index) || this.#cache.seen(entryPath, pattern, index + 1)) {
+          return;
+        }
+        const current = pattern.at(index);
+        const nextIndex = index + 1;
+        const next = pattern.at(nextIndex);
+        const fromSymlink = pattern.symlinks.has(index);
+
+        if (current === lazyMinimatch().GLOBSTAR) {
+          if (entry.name[0] === '.' || (this.#exclude && this.#exclude(entry.name))) {
+            continue;
+          }
+          if (!fromSymlink && entry.isDirectory()) {
+            // If directory, add ** to its potential patterns
+            subPatterns.add(index);
+          } else if (!fromSymlink && index === last) {
+            // If ** is last, add to results
+            if (!this.#results.has(entryPath)) {
+              this.#results.add(entryPath);
+              yield entryPath;
+            }
+          }
+
+          // Any pattern after ** is also a potential pattern
+          // so we can already test it here
+          const nextMatches = pattern.test(nextIndex, entry.name);
+          if (nextMatches && nextIndex === last && !isLast) {
+            // If next pattern is the last one, add to results
+            if (!this.#results.has(entryPath)) {
+              this.#results.add(entryPath);
+              yield entryPath;
+            }
+          } else if (nextMatches && entry.isDirectory()) {
+            // Pattern mached, meaning two patterns forward
+            // are also potential patterns
+            // e.g **/b/c when entry is a/b - add c to potential patterns
+            subPatterns.add(index + 2);
+          }
+          if ((nextMatches || pattern.at(0) === '.') &&
+           (entry.isDirectory() || entry.isSymbolicLink()) && !fromSymlink) {
+            // If pattern after ** matches, or pattern starts with "."
+            // and entry is a directory or symlink, add to potential patterns
+            subPatterns.add(nextIndex);
+          }
+
+          if (entry.isSymbolicLink()) {
+            nSymlinks.add(index);
+          }
+
+          if (next === '..' && entry.isDirectory()) {
+            // In case pattern is "**/..",
+            // both parent and current directory should be added to the queue
+            // if this is the last pattern, add to results instead
+            const parent = join(path, '..');
+            if (nextIndex < last) {
+              if (!this.#subpatterns.has(path) && !this.#cache.seen(path, pattern, nextIndex + 1)) {
+                this.#subpatterns.set(path, [pattern.child(new SafeSet().add(nextIndex + 1))]);
+              }
+              if (!this.#subpatterns.has(parent) && !this.#cache.seen(parent, pattern, nextIndex + 1)) {
+                this.#subpatterns.set(parent, [pattern.child(new SafeSet().add(nextIndex + 1))]);
+              }
+            } else {
+              if (!this.#cache.seen(path, pattern, nextIndex)) {
+                this.#cache.add(path, pattern.child(new SafeSet().add(nextIndex)));
+                if (!this.#results.has(path)) {
+                  this.#results.add(path);
+                  yield path;
+                }
+              }
+              if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) {
+                this.#cache.add(parent, pattern.child(new SafeSet().add(nextIndex)));
+                if (!this.#results.has(parent)) {
+                  this.#results.add(parent);
+                  yield parent;
+                }
+              }
+            }
+          }
+        }
+        if (typeof current === 'string') {
+          if (pattern.test(index, entry.name) && index !== last) {
+            // If current pattern matches entry name
+            // the next pattern is a potential pattern
+            subPatterns.add(nextIndex);
+          } else if (current === '.' && pattern.test(nextIndex, entry.name)) {
+            // If current pattern is ".", proceed to test next pattern
+            if (nextIndex === last) {
+              if (!this.#results.has(entryPath)) {
+                this.#results.add(entryPath);
+                yield entryPath;
+              }
+            } else {
+              subPatterns.add(nextIndex + 1);
+            }
+          }
+        }
+        if (typeof current === 'object' && pattern.test(index, entry.name)) {
+          // If current pattern is a regex that matches entry name (e.g *.js)
+          // add next pattern to potential patterns, or to results if it's the last pattern
+          if (index === last) {
+            if (!this.#results.has(entryPath)) {
+              this.#results.add(entryPath);
+              yield entryPath;
+            }
+          } else if (entry.isDirectory()) {
+            subPatterns.add(nextIndex);
+          }
+        }
+      }
+      if (subPatterns.size > 0) {
+        // If there are potential patterns, add to queue
+        this.#addSubpattern(entryPath, pattern.child(subPatterns, nSymlinks));
+      }
+    }
+  }
 }
 
 module.exports = {

lib/internal/fs/promises.js

@@ -90,10 +90,11 @@ const pathModule = require('path');
 const { isAbsolute } = pathModule;
 const { toPathIfFileURL } = require('internal/url');
 const {
+  emitExperimentalWarning,
+  getLazy,
   kEmptyObject,
   lazyDOMException,
   promisify,
-  getLazy,
 } = require('internal/util');
 const EventEmitter = require('events');
 const { StringDecoder } = require('string_decoder');
@@ -1267,11 +1268,19 @@ async function* _watch(filename, options = kEmptyObject) {
   yield* watch(filename, options);
 }
 
+const lazyGlob = getLazy(() => require('internal/fs/glob').Glob);
+async function* glob(pattern, options) {
+  emitExperimentalWarning('glob');
+  const Glob = lazyGlob();
+  yield* new Glob(pattern, options).glob();
+}
+
 module.exports = {
   exports: {
     access,
     copyFile,
     cp,
+    glob,
     open,
     opendir: promisify(opendir),
     rename,

test/parallel/test-fs-glob.mjs

@@ -1,11 +1,11 @@
-// Flags: --expose-internals
 import * as common from '../common/index.mjs';
 import tmpdir from '../common/tmpdir.js';
 import { resolve, dirname, sep } from 'node:path';
-import { mkdir, writeFile, symlink } from 'node:fs/promises';
+import { mkdir, writeFile, symlink, glob as asyncGlob } from 'node:fs/promises';
-import { test } from 'node:test';
+import { glob, globSync } from 'node:fs';
+import { test, describe } from 'node:test';
+import { promisify } from 'node:util';
 import assert from 'node:assert';
-import glob from 'internal/fs/glob';
 
 tmpdir.refresh();
 
@@ -301,10 +301,35 @@ const patterns = {
   ],
 };
 
+describe('glob', function() {
+  const promisified = promisify(glob);
   for (const [pattern, expected] of Object.entries(patterns)) {
-  test(pattern, () => {
+    test(pattern, async () => {
-    const actual = new glob.Glob([pattern], { cwd: fixtureDir }).globSync().sort();
+      const actual = (await promisified(pattern, { cwd: fixtureDir })).sort();
       const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
       assert.deepStrictEqual(actual, normalized);
     });
   }
+});
+
+describe('globSync', function() {
+  for (const [pattern, expected] of Object.entries(patterns)) {
+    test(pattern, () => {
+      const actual = globSync(pattern, { cwd: fixtureDir }).sort();
+      const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
+      assert.deepStrictEqual(actual, normalized);
+    });
+  }
+});
+
+describe('fsPromises glob', function() {
+  for (const [pattern, expected] of Object.entries(patterns)) {
+    test(pattern, async () => {
+      const actual = [];
+      for await (const item of asyncGlob(pattern, { cwd: fixtureDir })) actual.push(item);
+      actual.sort();
+      const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort();
+      assert.deepStrictEqual(actual, normalized);
+    });
+  }
+});

test/parallel/test-permission-fs-supported.js

@@ -90,6 +90,7 @@ const ignoreList = [
   ...syncAndAsyncAPI('readv'),
   ...syncAndAsyncAPI('write'),
   ...syncAndAsyncAPI('writev'),
+  ...syncAndAsyncAPI('glob'),
 ];
 
 {