From d96a9a94f5fc36c5668ab71ffdc73033d325f880 Mon Sep 17 00:00:00 2001 From: Graham Campbell Date: Wed, 22 Apr 2026 16:54:46 +0100 Subject: [PATCH 1/3] Replace various old packages --- components/framework/index.js | 2 +- package.json | 10 +- src/state/S3StateStorage.js | 8 +- src/utils/glob.js | 151 +++++++++++++++++++ test/unit/components/framework/index.test.js | 55 +++++++ test/unit/src/state/S3StateStorage.test.js | 52 +++++++ 6 files changed, 266 insertions(+), 12 deletions(-) create mode 100644 src/utils/glob.js diff --git a/components/framework/index.js b/components/framework/index.js index 0bacc2a..f4e060b 100644 --- a/components/framework/index.js +++ b/components/framework/index.js @@ -3,7 +3,7 @@ const spawn = require('cross-spawn'); const YAML = require('js-yaml'); const hasha = require('hasha'); -const globby = require('globby'); +const globby = require('../../src/utils/glob'); const path = require('path'); const spawnExt = require('child-process-ext/spawn'); const semver = require('semver'); diff --git a/package.json b/package.json index 6448011..824f21e 100644 --- a/package.json +++ b/package.json @@ -35,14 +35,14 @@ "chalk": "^4.1.2", "child-process-ext": "^2.1.1", "ci-info": "^3.3.2", - "cli-progress-footer": "^2.3.2", "cli-cursor": "^3", - "d": "^1.0.1", + "cli-progress-footer": "^2.3.2", "cross-spawn": "^7.0.3", + "d": "^1.0.1", "event-emitter": "^0.3.5", "ext": "^1.7.0", + "fast-glob": "^3.3.3", "fs-extra": "^10.1.0", - "globby": "^11.1.0", "hasha": "^5.2.2", "is-interactive": "^1", "is-unicode-supported": "^0.1", @@ -53,7 +53,6 @@ "minimist": "^1.2.6", "p-limit": "^3.1.0", "path2": "^0.1.0", - "promise-queue": "^2.2.5", "ramda": "^0.28.0", "semver": "^7.3.7", "signal-exit": "^3.0.7", @@ -61,8 +60,7 @@ "supports-color": "^8.1.1", "traverse": "^0.6.6", "type": "^2.6.1", - "uni-global": "^1.0.0", - "uuid": "^8.3.2" + "uni-global": "^1.0.0" }, "devDependencies": { "@types/mocha": "^9.1.1", diff --git a/src/state/S3StateStorage.js b/src/state/S3StateStorage.js index afd32b4..aef56b7 100644 --- a/src/state/S3StateStorage.js +++ b/src/state/S3StateStorage.js @@ -1,13 +1,11 @@ 'use strict'; const { S3 } = require('@aws-sdk/client-s3'); -const PromiseQueue = require('promise-queue'); +const pLimit = require('p-limit'); const streamToString = require('../utils/stream-to-string'); const ServerlessError = require('../serverless-error'); const BaseStateStorage = require('./BaseStateStorage'); -PromiseQueue.configure(Promise); - class S3StateStorage extends BaseStateStorage { constructor(config = {}) { super(); @@ -20,7 +18,7 @@ class S3StateStorage extends BaseStateStorage { this.s3Client = new S3({ region: this.region, credentials: config.credentials }); - this.writeRequestQueue = new PromiseQueue(1, Infinity); + this.writeRequestQueue = pLimit(1); } async readState() { @@ -52,7 +50,7 @@ class S3StateStorage extends BaseStateStorage { async writeState() { try { - await this.writeRequestQueue.add(async () => { + await this.writeRequestQueue(async () => { await this.s3Client.putObject({ Bucket: this.bucketName, Key: this.stateKey, diff --git a/src/utils/glob.js b/src/utils/glob.js new file mode 100644 index 0000000..59a3112 --- /dev/null +++ b/src/utils/glob.js @@ -0,0 +1,151 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const fastGlob = require('fast-glob'); + +const toArray = (patterns) => { + return Array.isArray(patterns) ? patterns : [patterns]; +}; + +const isDynamicPattern = (pattern) => /[*?{}()[\]]/.test(pattern); + +const maybeExpandDirectoryPattern = (pattern, cwd) => { + if (isDynamicPattern(pattern)) { + return pattern; + } + + const absolutePattern = path.resolve(cwd || process.cwd(), pattern); + try { + if (fs.statSync(absolutePattern).isDirectory()) { + const normalizedPattern = pattern.replace(/\\/g, '/').replace(/\/$/, ''); + return `${normalizedPattern}/**/*`; + } + } catch { + // Ignore missing paths and let fast-glob handle them. + } + + return pattern; +}; + +const expandPattern = (pattern, cwd, expandDirectories) => { + if (!expandDirectories) { + return pattern; + } + + if (pattern.startsWith('!')) { + return `!${maybeExpandDirectoryPattern(pattern.slice(1), cwd)}`; + } + + return maybeExpandDirectoryPattern(pattern, cwd); +}; + +const normalizeOptions = (options = {}) => { + const globOptions = { ...options }; + const expandDirectories = + globOptions.expandDirectories !== undefined ? globOptions.expandDirectories : true; + + delete globOptions.expandDirectories; + delete globOptions.nosort; + + if (globOptions.follow !== undefined) { + globOptions.followSymbolicLinks = globOptions.follow; + delete globOptions.follow; + } + + if (globOptions.nodir !== undefined) { + globOptions.onlyFiles = globOptions.nodir; + delete globOptions.nodir; + } + + if (globOptions.silent !== undefined) { + globOptions.suppressErrors = globOptions.silent; + delete globOptions.silent; + } + + return { expandDirectories, globOptions }; +}; + +const buildTasks = (patterns, globOptions) => { + const tasks = []; + + for (let index = 0; index < patterns.length; index += 1) { + const pattern = patterns[index]; + if (pattern.startsWith('!')) { + continue; + } + + const ignore = patterns + .slice(index) + .filter((value) => value.startsWith('!')) + .map((value) => value.slice(1)); + + tasks.push({ + pattern, + options: { + ...globOptions, + ignore: [...(globOptions.ignore || []), ...ignore], + }, + }); + } + + return tasks; +}; + +const collectResults = async (tasks) => { + const seen = new Set(); + const results = []; + + for (const task of tasks) { + for (const entry of await fastGlob(task.pattern, task.options)) { + if (seen.has(entry)) { + continue; + } + + seen.add(entry); + results.push(entry); + } + } + + return results; +}; + +const collectResultsSync = (tasks) => { + const seen = new Set(); + const results = []; + + for (const task of tasks) { + for (const entry of fastGlob.sync(task.pattern, task.options)) { + if (seen.has(entry)) { + continue; + } + + seen.add(entry); + results.push(entry); + } + } + + return results; +}; + +const glob = async (patterns, options = {}) => { + const normalizedPatterns = toArray(patterns); + const { expandDirectories, globOptions } = normalizeOptions(options); + const expandedPatterns = normalizedPatterns.map((pattern) => + expandPattern(pattern, globOptions.cwd, expandDirectories) + ); + + return collectResults(buildTasks(expandedPatterns, globOptions)); +}; + +glob.sync = (patterns, options = {}) => { + const normalizedPatterns = toArray(patterns); + const { expandDirectories, globOptions } = normalizeOptions(options); + const expandedPatterns = normalizedPatterns.map((pattern) => + expandPattern(pattern, globOptions.cwd, expandDirectories) + ); + + return collectResultsSync(buildTasks(expandedPatterns, globOptions)); +}; + +module.exports = glob; diff --git a/test/unit/components/framework/index.test.js b/test/unit/components/framework/index.test.js index 716c0ce..7a8c4b0 100644 --- a/test/unit/components/framework/index.test.js +++ b/test/unit/components/framework/index.test.js @@ -1,7 +1,10 @@ 'use strict'; +const fs = require('node:fs').promises; +const path = require('path'); const proxyquire = require('proxyquire'); const chai = require('chai'); +const fse = require('fs-extra'); const sinon = require('sinon'); const Context = require('../../../../src/Context'); const ComponentContext = require('../../../../src/ComponentContext'); @@ -545,4 +548,56 @@ describe('test/unit/components/framework/index.test.js', () => { .to.throw() .and.have.property('code', 'INVALID_PATH_IN_SERVICE_CONFIGURATION'); }); + + it('expands literal directory cache patterns when calculating hashes', async () => { + const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-dir-')); + + try { + await fse.outputFile(path.join(serviceDir, 'src', 'handler.js'), 'module.exports = 1;\n'); + + const context = await getContext(); + const directoryPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['src'], + }); + const globPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['src/**/*'], + }); + + expect(await directoryPatternComponent.calculateCacheHash()).to.equal( + await globPatternComponent.calculateCacheHash() + ); + } finally { + await fse.remove(serviceDir); + } + }); + + it('supports negated cache patterns that re-include a later file', async () => { + const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-negation-')); + + try { + await Promise.all([ + fse.outputFile(path.join(serviceDir, 'keep.js'), 'keep\n'), + fse.outputFile(path.join(serviceDir, 'ignored', 'drop.js'), 'drop\n'), + fse.outputFile(path.join(serviceDir, 'ignored', 'reinclude.js'), 'reinclude\n'), + ]); + + const context = await getContext(); + const negatedPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['**/*', '!ignored/**/*', 'ignored/reinclude.js'], + }); + const explicitPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['keep.js', 'ignored/reinclude.js'], + }); + + expect(await negatedPatternComponent.calculateCacheHash()).to.equal( + await explicitPatternComponent.calculateCacheHash() + ); + } finally { + await fse.remove(serviceDir); + } + }); }); diff --git a/test/unit/src/state/S3StateStorage.test.js b/test/unit/src/state/S3StateStorage.test.js index 3dfd721..ce67b72 100644 --- a/test/unit/src/state/S3StateStorage.test.js +++ b/test/unit/src/state/S3StateStorage.test.js @@ -9,6 +9,17 @@ const S3StateStorage = require('../../../../src/state/S3StateStorage'); const expect = chai.expect; +const createDeferred = () => { + let resolve; + let reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + + return { promise, resolve, reject }; +}; + describe('test/unit/src/state/S3StateStorage.test.js', () => { const bucketName = 'dummy-bucket'; const stateKey = 'dummy-key'; @@ -108,4 +119,45 @@ describe('test/unit/src/state/S3StateStorage.test.js', () => { credentials: 'creds', }); }); + + it('serializes concurrent state writes', async () => { + const firstWrite = createDeferred(); + const secondWrite = createDeferred(); + let activeWrites = 0; + let maxActiveWrites = 0; + const s3StateStorage = new S3StateStorage({ bucketName, stateKey }); + const mockedS3Client = { + putObject: sinon.stub().callsFake(() => { + activeWrites += 1; + maxActiveWrites = Math.max(maxActiveWrites, activeWrites); + const currentWrite = mockedS3Client.putObject.callCount === 1 ? firstWrite : secondWrite; + return currentWrite.promise.finally(() => { + activeWrites -= 1; + }); + }), + }; + + s3StateStorage.s3Client = mockedS3Client; + s3StateStorage.state = { first: true }; + + const firstPromise = s3StateStorage.writeState(); + s3StateStorage.state = { second: true }; + const secondPromise = s3StateStorage.writeState(); + + await Promise.resolve(); + await Promise.resolve(); + + expect(mockedS3Client.putObject.calledOnce).to.equal(true); + + firstWrite.resolve(); + await firstPromise; + await Promise.resolve(); + + expect(mockedS3Client.putObject.calledTwice).to.equal(true); + + secondWrite.resolve(); + await Promise.all([firstPromise, secondPromise]); + + expect(maxActiveWrites).to.equal(1); + }); }); From f98b2c19fb3fbbd278da2c9f39673bf0a0e2bc7d Mon Sep 17 00:00:00 2001 From: Graham Campbell Date: Thu, 23 Apr 2026 00:29:42 +0100 Subject: [PATCH 2/3] Apply corrections from code review --- src/utils/glob.js | 11 ++++++++++- test/unit/src/utils/glob.test.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 test/unit/src/utils/glob.test.js diff --git a/src/utils/glob.js b/src/utils/glob.js index 59a3112..314412c 100644 --- a/src/utils/glob.js +++ b/src/utils/glob.js @@ -68,6 +68,15 @@ const normalizeOptions = (options = {}) => { const buildTasks = (patterns, globOptions) => { const tasks = []; + const leadingIgnore = []; + + for (const pattern of patterns) { + if (!pattern.startsWith('!')) { + break; + } + + leadingIgnore.push(pattern.slice(1)); + } for (let index = 0; index < patterns.length; index += 1) { const pattern = patterns[index]; @@ -84,7 +93,7 @@ const buildTasks = (patterns, globOptions) => { pattern, options: { ...globOptions, - ignore: [...(globOptions.ignore || []), ...ignore], + ignore: [...(globOptions.ignore || []), ...(tasks.length ? [] : leadingIgnore), ...ignore], }, }); } diff --git a/test/unit/src/utils/glob.test.js b/test/unit/src/utils/glob.test.js new file mode 100644 index 0000000..7d1bb79 --- /dev/null +++ b/test/unit/src/utils/glob.test.js @@ -0,0 +1,31 @@ +'use strict'; + +const fs = require('node:fs').promises; +const os = require('node:os'); +const path = require('node:path'); +const fse = require('fs-extra'); +const { expect } = require('chai'); + +const glob = require('../../../../src/utils/glob'); + +describe('test/unit/src/utils/glob.test.js', () => { + let tmpDir; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'compose-glob-')); + }); + + afterEach(async () => { + await fse.remove(tmpDir); + }); + + it('honors leading negation patterns for async and sync calls', async () => { + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'keep.js'), 'keep\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'drop.js'), 'drop\n'), + ]); + + expect(await glob(['!ignored/**/*', '**/*.js'], { cwd: tmpDir })).to.deep.equal(['keep.js']); + expect(glob.sync(['!ignored/**/*', '**/*.js'], { cwd: tmpDir })).to.deep.equal(['keep.js']); + }); +}); From c300cb9aede3d1be0e410d1e3df6773c8ab5c0cd Mon Sep 17 00:00:00 2001 From: Graham Campbell Date: Thu, 23 Apr 2026 01:26:52 +0100 Subject: [PATCH 3/3] Applied more review feedback --- components/framework/index.js | 4 +-- src/utils/glob.js | 53 +++++++++++++++++--------------- test/unit/src/utils/glob.test.js | 27 ++++++++++++++-- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/components/framework/index.js b/components/framework/index.js index f4e060b..bb2c71e 100644 --- a/components/framework/index.js +++ b/components/framework/index.js @@ -3,7 +3,7 @@ const spawn = require('cross-spawn'); const YAML = require('js-yaml'); const hasha = require('hasha'); -const globby = require('../../src/utils/glob'); +const glob = require('../../src/utils/glob'); const path = require('path'); const spawnExt = require('child-process-ext/spawn'); const semver = require('semver'); @@ -316,7 +316,7 @@ class ServerlessFramework { async calculateCacheHash() { const algorithm = 'md5'; // fastest - const allFilePaths = await globby(this.inputs.cachePatterns, { + const allFilePaths = await glob(this.inputs.cachePatterns, { cwd: this.inputs.path, }); diff --git a/src/utils/glob.js b/src/utils/glob.js index 314412c..cc2568c 100644 --- a/src/utils/glob.js +++ b/src/utils/glob.js @@ -68,34 +68,39 @@ const normalizeOptions = (options = {}) => { const buildTasks = (patterns, globOptions) => { const tasks = []; - const leadingIgnore = []; - - for (const pattern of patterns) { - if (!pattern.startsWith('!')) { + let remainingPatterns = patterns; + + while (remainingPatterns.length > 0) { + const negativePatternIndex = remainingPatterns.findIndex((pattern) => pattern.startsWith('!')); + + if (negativePatternIndex === -1) { + tasks.push({ + patterns: remainingPatterns, + options: { + ...globOptions, + ignore: [...(globOptions.ignore || [])], + }, + }); break; } - leadingIgnore.push(pattern.slice(1)); - } + const ignorePattern = remainingPatterns[negativePatternIndex].slice(1); + + for (const task of tasks) { + task.options.ignore.push(ignorePattern); + } - for (let index = 0; index < patterns.length; index += 1) { - const pattern = patterns[index]; - if (pattern.startsWith('!')) { - continue; + if (negativePatternIndex !== 0) { + tasks.push({ + patterns: remainingPatterns.slice(0, negativePatternIndex), + options: { + ...globOptions, + ignore: [...(globOptions.ignore || []), ignorePattern], + }, + }); } - const ignore = patterns - .slice(index) - .filter((value) => value.startsWith('!')) - .map((value) => value.slice(1)); - - tasks.push({ - pattern, - options: { - ...globOptions, - ignore: [...(globOptions.ignore || []), ...(tasks.length ? [] : leadingIgnore), ...ignore], - }, - }); + remainingPatterns = remainingPatterns.slice(negativePatternIndex + 1); } return tasks; @@ -106,7 +111,7 @@ const collectResults = async (tasks) => { const results = []; for (const task of tasks) { - for (const entry of await fastGlob(task.pattern, task.options)) { + for (const entry of await fastGlob(task.patterns, task.options)) { if (seen.has(entry)) { continue; } @@ -124,7 +129,7 @@ const collectResultsSync = (tasks) => { const results = []; for (const task of tasks) { - for (const entry of fastGlob.sync(task.pattern, task.options)) { + for (const entry of fastGlob.sync(task.patterns, task.options)) { if (seen.has(entry)) { continue; } diff --git a/test/unit/src/utils/glob.test.js b/test/unit/src/utils/glob.test.js index 7d1bb79..dafd1c3 100644 --- a/test/unit/src/utils/glob.test.js +++ b/test/unit/src/utils/glob.test.js @@ -19,13 +19,34 @@ describe('test/unit/src/utils/glob.test.js', () => { await fse.remove(tmpDir); }); - it('honors leading negation patterns for async and sync calls', async () => { + it('matches globby order-sensitive leading negation behavior for async and sync calls', async () => { await Promise.all([ fse.outputFile(path.join(tmpDir, 'keep.js'), 'keep\n'), + fse.outputFile(path.join(tmpDir, 'keep.ts'), 'keep\n'), fse.outputFile(path.join(tmpDir, 'ignored', 'drop.js'), 'drop\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'drop.ts'), 'drop\n'), ]); - expect(await glob(['!ignored/**/*', '**/*.js'], { cwd: tmpDir })).to.deep.equal(['keep.js']); - expect(glob.sync(['!ignored/**/*', '**/*.js'], { cwd: tmpDir })).to.deep.equal(['keep.js']); + expect( + (await glob(['!ignored/**/*', '**/*.js', '**/*.ts'], { cwd: tmpDir })).sort() + ).to.deep.equal(['ignored/drop.js', 'ignored/drop.ts', 'keep.js', 'keep.ts']); + expect( + glob.sync(['!ignored/**/*', '**/*.js', '**/*.ts'], { cwd: tmpDir }).sort() + ).to.deep.equal(['ignored/drop.js', 'ignored/drop.ts', 'keep.js', 'keep.ts']); + }); + + it('applies later negations to earlier positives and still supports re-inclusion', async () => { + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'keep.js'), 'keep\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'drop.js'), 'drop\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'reinclude.js'), 'reinclude\n'), + ]); + + expect( + (await glob(['**/*.js', '!ignored/**/*', 'ignored/reinclude.js'], { cwd: tmpDir })).sort() + ).to.deep.equal(['ignored/reinclude.js', 'keep.js']); + expect( + glob.sync(['**/*.js', '!ignored/**/*', 'ignored/reinclude.js'], { cwd: tmpDir }).sort() + ).to.deep.equal(['ignored/reinclude.js', 'keep.js']); }); });