From fadc798bf814f554bfc043b3b6a1eeed1ad9c7aa Mon Sep 17 00:00:00 2001 From: Philippe Serhal Date: Tue, 18 Feb 2025 16:40:19 -0500 Subject: [PATCH 1/3] fix(deps): replace deprecated glob@8 Installing netlify-cli prints a deprecation warning: https://github.com/netlify/cli/issues/7029. I was going to upgrade `glob`, but then I remembered it's a fairly large library, so why not use this as an opportunity to replace it? See https://github.com/es-tooling/module-replacements/blob/main/docs/modules/glob.md. Compare the APIs: - https://github.com/isaacs/node-glob#readme - https://github.com/SuperchupuDev/tinyglobby?tab=readme-ov-file#api The only differences are option names, which I've updated. Our uses either hardcode patterns or enforce precisely documented user patterns, so we won't hit any of the limitations of tinyglobby. --- package-lock.json | 64 +++++-------------- packages/zip-it-and-ship-it/package.json | 3 +- .../runtimes/node/bundlers/esbuild/bundler.ts | 8 +-- .../runtimes/node/bundlers/nft/side_files.ts | 2 +- .../runtimes/node/bundlers/zisi/published.ts | 2 +- .../runtimes/node/bundlers/zisi/tree_files.ts | 2 +- .../zip-it-and-ship-it/src/utils/matching.ts | 12 ++-- .../zip-it-and-ship-it/tests/v2api.test.ts | 10 +-- 8 files changed, 33 insertions(+), 70 deletions(-) diff --git a/package-lock.json b/package-lock.json index db18a41477..5ac871725f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9086,28 +9086,12 @@ "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", "license": "MIT" }, - "node_modules/@types/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/@types/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==", - "dev": true, - "dependencies": { - "@types/minimatch": "^5.1.2", - "@types/node": "*" - } - }, "node_modules/@types/glob-to-regexp": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/@types/glob-to-regexp/-/glob-to-regexp-0.4.4.tgz", "integrity": "sha512-nDKoaKJYbnn1MZxUY0cA1bPmmgZbg0cTq7Rh13d0KWYNOiKbqoR+2d89SnRPszGh7ROzSwZ/GOjZ4jPbmmZ6Eg==", "dev": true }, - "node_modules/@types/glob/node_modules/@types/minimatch": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", - "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==", - "dev": true - }, "node_modules/@types/http-cache-semantics": { "version": "4.0.4", "license": "MIT" @@ -16333,7 +16317,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -25748,6 +25731,21 @@ "dev": true, "license": "MIT" }, + "node_modules/tinyglobby": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.12.tgz", + "integrity": "sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==", + "dependencies": { + "fdir": "^6.4.3", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, "node_modules/tinypool": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.7.0.tgz", @@ -30118,7 +30116,6 @@ "fast-glob": "^3.3.2", "filter-obj": "^5.0.0", "find-up": "^6.0.0", - "glob": "^8.0.3", "is-builtin-module": "^3.1.0", "is-path-inside": "^4.0.0", "junk": "^4.0.0", @@ -30132,6 +30129,7 @@ "require-package-name": "^2.0.1", "resolve": "^2.0.0-next.1", "semver": "^7.3.8", + "tinyglobby": "^0.2.12", "tmp-promise": "^3.0.2", "toml": "^3.0.0", "unixify": "^1.0.0", @@ -30144,7 +30142,6 @@ }, "devDependencies": { "@types/archiver": "6.0.3", - "@types/glob": "8.1.0", "@types/is-ci": "3.0.4", "@types/node": "20.12.11", "@types/normalize-path": "3.0.2", @@ -30608,35 +30605,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "packages/zip-it-and-ship-it/node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "packages/zip-it-and-ship-it/node_modules/glob/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "packages/zip-it-and-ship-it/node_modules/hosted-git-info": { "version": "2.8.9", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", diff --git a/packages/zip-it-and-ship-it/package.json b/packages/zip-it-and-ship-it/package.json index 0b3d4c642f..c31fffb8cd 100644 --- a/packages/zip-it-and-ship-it/package.json +++ b/packages/zip-it-and-ship-it/package.json @@ -55,7 +55,6 @@ "fast-glob": "^3.3.2", "filter-obj": "^5.0.0", "find-up": "^6.0.0", - "glob": "^8.0.3", "is-builtin-module": "^3.1.0", "is-path-inside": "^4.0.0", "junk": "^4.0.0", @@ -69,6 +68,7 @@ "require-package-name": "^2.0.1", "resolve": "^2.0.0-next.1", "semver": "^7.3.8", + "tinyglobby": "^0.2.12", "tmp-promise": "^3.0.2", "toml": "^3.0.0", "unixify": "^1.0.0", @@ -78,7 +78,6 @@ }, "devDependencies": { "@types/archiver": "6.0.3", - "@types/glob": "8.1.0", "@types/is-ci": "3.0.4", "@types/node": "20.12.11", "@types/normalize-path": "3.0.2", diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/esbuild/bundler.ts b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/esbuild/bundler.ts index bb639427be..b0d6133f3f 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/esbuild/bundler.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/esbuild/bundler.ts @@ -40,7 +40,7 @@ const includedFilesToEsbuildExternals = async (includedFiles: string[], baseDir: .map((pattern) => pattern.slice(1)) // esbuild expects relative paths .map((pattern) => `./${pattern}`) - // esbuild treats * the same as glob treats **, so this replacement is safe + // esbuild treats * the same as tinyglobby treats **, so this replacement is safe .map((pattern) => pattern.replace(/\*\*/g, '*').replace(/\*(\\\*)+/g, '*')) const result: string[] = [] @@ -60,11 +60,11 @@ const includedFilesToEsbuildExternals = async (includedFiles: string[], baseDir: if (hasMultipleGlobs) { const resolved = await glob(pattern, { - noglobstar: true, cwd: baseDir, }) - - result.push(...resolved) + // esbuild expects relative paths, but tinyglobby uses `posix.normalize()` which strips leading `./` + const esbuildPatterns = resolved.map((pattern) => `./${pattern}`) + result.push(...esbuildPatterns) } else { result.push(pattern) } diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/nft/side_files.ts b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/nft/side_files.ts index 61c46e3fb7..2e41dcaed9 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/nft/side_files.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/nft/side_files.ts @@ -18,7 +18,7 @@ export const getSideFiles = async function (functionPath: string, stat: Stats): absolute: true, cwd: functionPath, ignore: `**/node_modules/**`, - nodir: true, + onlyFiles: true, }) return paths.filter((path) => !isJunk(basename(path))) diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts index ab977b5e22..e599714a78 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts @@ -5,7 +5,7 @@ export const getPublishedFiles = async function (modulePath: string): Promise { - let normalizedIgnore +export const glob = function (pattern: string, options: GlobOptions): Promise { + let normalizedIgnore: undefined | string | string[] if (options.ignore) { normalizedIgnore = @@ -22,7 +18,7 @@ export const glob = function (pattern: string, options: globFunction.IOptions): : options.ignore.map((expression) => normalizePath(expression)) } - return pGlob(normalizePath(pattern), { ...options, ignore: normalizedIgnore }) + return tinyGlobby(normalizePath(pattern), { ...options, ignore: normalizedIgnore }) } export const minimatch = function (target: string, pattern: string, options?: MinimatchOptions): boolean { diff --git a/packages/zip-it-and-ship-it/tests/v2api.test.ts b/packages/zip-it-and-ship-it/tests/v2api.test.ts index fbd323cb06..0a788c6e96 100644 --- a/packages/zip-it-and-ship-it/tests/v2api.test.ts +++ b/packages/zip-it-and-ship-it/tests/v2api.test.ts @@ -1,13 +1,12 @@ import { readFile } from 'fs/promises' import { join, resolve } from 'path' import { platform, version as nodeVersion } from 'process' -import { promisify } from 'util' import { getPath as getBootstrapPath } from '@netlify/serverless-functions-api' import merge from 'deepmerge' -import glob from 'glob' import { pathExists } from 'path-exists' import semver from 'semver' +import { glob } from 'tinyglobby' import { dir as getTmpDir } from 'tmp-promise' import { afterEach, describe, expect, test, vi } from 'vitest' @@ -18,8 +17,6 @@ import { invokeLambda, readAsBuffer } from './helpers/lambda.js' import { zipFixture, unzipFiles, importFunctionFile, FIXTURES_ESM_DIR, FIXTURES_DIR } from './helpers/main.js' import { testMany } from './helpers/test_many.js' -const pGlob = promisify(glob) - vi.mock('../src/utils/shell.js', () => ({ shellUtils: { runCommand: vi.fn() } })) describe.runIf(semver.gte(nodeVersion, '18.13.0'))('V2 functions API', () => { @@ -132,7 +129,10 @@ describe.runIf(semver.gte(nodeVersion, '18.13.0'))('V2 functions API', () => { const [{ name: archive, entryFilename, path }] = files - const untranspiledFiles = await pGlob(`${path}/**/*.ts`) + const untranspiledFiles = await glob(`${path}/**/*.ts`, { + onlyFiles: true, + followSymbolicLinks: false, + }) expect(untranspiledFiles).toEqual([]) const func = await importFunctionFile(`${tmpDir}/${archive}/${entryFilename}`) From bac57015b5a1630d8f9751bb892b3cbc7a595034 Mon Sep 17 00:00:00 2001 From: Philippe Serhal Date: Tue, 18 Feb 2025 16:56:28 -0500 Subject: [PATCH 2/3] fix(deps): replace fast-glob with tinyglobby --- package-lock.json | 2 +- packages/zip-it-and-ship-it/package.json | 2 +- .../src/runtimes/node/utils/included_files.ts | 3 ++- packages/zip-it-and-ship-it/tests/main.test.ts | 2 +- packages/zip-it-and-ship-it/tests/telemetry.test.ts | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5ac871725f..55179a3ccb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30113,7 +30113,7 @@ "es-module-lexer": "^1.0.0", "esbuild": "0.19.11", "execa": "^7.0.0", - "fast-glob": "^3.3.2", + "fast-glob": "^3.3.3", "filter-obj": "^5.0.0", "find-up": "^6.0.0", "is-builtin-module": "^3.1.0", diff --git a/packages/zip-it-and-ship-it/package.json b/packages/zip-it-and-ship-it/package.json index c31fffb8cd..4c2280b403 100644 --- a/packages/zip-it-and-ship-it/package.json +++ b/packages/zip-it-and-ship-it/package.json @@ -52,7 +52,7 @@ "es-module-lexer": "^1.0.0", "esbuild": "0.19.11", "execa": "^7.0.0", - "fast-glob": "^3.3.2", + "fast-glob": "^3.3.3", "filter-obj": "^5.0.0", "find-up": "^6.0.0", "is-builtin-module": "^3.1.0", diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/utils/included_files.ts b/packages/zip-it-and-ship-it/src/runtimes/node/utils/included_files.ts index ef4e543da6..1d625930b4 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/utils/included_files.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/utils/included_files.ts @@ -56,12 +56,13 @@ export const getPathsOfIncludedFiles = async ( onlyFiles: false, // get directories as well to get symlinked directories, // to filter the regular non symlinked directories out mark them with a slash at the end to filter them out. + // TODO(serhalp) If we can find a better way to do this, we can use tinyglobby here and remove fast-glob entirely. markDirectories: true, followSymbolicLinks: false, }) + // now filter the non symlinked directories out that got marked with a trailing slash const paths = pathGroups.filter((path) => !path.endsWith('/')).map(normalize) - // now filter the non symlinked directories out that got marked with a trailing slash return { excludePatterns, paths } } diff --git a/packages/zip-it-and-ship-it/tests/main.test.ts b/packages/zip-it-and-ship-it/tests/main.test.ts index e25a1c99bc..28ef1836c0 100644 --- a/packages/zip-it-and-ship-it/tests/main.test.ts +++ b/packages/zip-it-and-ship-it/tests/main.test.ts @@ -6,10 +6,10 @@ import cpy from 'cpy' import decompress from 'decompress' import merge from 'deepmerge' import { execa, execaNode } from 'execa' -import glob from 'fast-glob' import isCI from 'is-ci' import { pathExists } from 'path-exists' import semver from 'semver' +import { glob } from 'tinyglobby' import { dir as getTmpDir, tmpName } from 'tmp-promise' import unixify from 'unixify' import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from 'vitest' diff --git a/packages/zip-it-and-ship-it/tests/telemetry.test.ts b/packages/zip-it-and-ship-it/tests/telemetry.test.ts index cb76676740..82492c99e1 100644 --- a/packages/zip-it-and-ship-it/tests/telemetry.test.ts +++ b/packages/zip-it-and-ship-it/tests/telemetry.test.ts @@ -1,7 +1,7 @@ import { join } from 'path' import decompress from 'decompress' -import glob from 'fast-glob' +import { glob } from 'tinyglobby' import { dir as getTmpDir } from 'tmp-promise' import { expect, test } from 'vitest' From 3514a13f9d9b06a0c197cb011232857946670280 Mon Sep 17 00:00:00 2001 From: Philippe Serhal Date: Fri, 28 Feb 2025 17:07:33 -0500 Subject: [PATCH 3/3] chore(wip): ??? at least it doesn't hang now --- .../src/runtimes/node/bundlers/zisi/published.ts | 1 + .../src/runtimes/node/bundlers/zisi/tree_files.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts index e599714a78..da12b20552 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/bundlers/zisi/published.ts @@ -5,6 +5,7 @@ export const getPublishedFiles = async function (modulePath: string): Promise