diff --git a/components/framework/index.js b/components/framework/index.js index 0bacc2a..bb2c71e 100644 --- a/components/framework/index.js +++ b/components/framework/index.js @@ -3,7 +3,7 @@ const spawn = require('cross-spawn'); const YAML = require('js-yaml'); const hasha = require('hasha'); -const globby = require('globby'); +const glob = require('../../src/utils/glob'); const path = require('path'); const spawnExt = require('child-process-ext/spawn'); const semver = require('semver'); @@ -316,7 +316,7 @@ class ServerlessFramework { async calculateCacheHash() { const algorithm = 'md5'; // fastest - const allFilePaths = await globby(this.inputs.cachePatterns, { + const allFilePaths = await glob(this.inputs.cachePatterns, { cwd: this.inputs.path, }); diff --git a/package.json b/package.json index 6448011..824f21e 100644 --- a/package.json +++ b/package.json @@ -35,14 +35,14 @@ "chalk": "^4.1.2", "child-process-ext": "^2.1.1", "ci-info": "^3.3.2", - "cli-progress-footer": "^2.3.2", "cli-cursor": "^3", - "d": "^1.0.1", + "cli-progress-footer": "^2.3.2", "cross-spawn": "^7.0.3", + "d": "^1.0.1", "event-emitter": "^0.3.5", "ext": "^1.7.0", + "fast-glob": "^3.3.3", "fs-extra": "^10.1.0", - "globby": "^11.1.0", "hasha": "^5.2.2", "is-interactive": "^1", "is-unicode-supported": "^0.1", @@ -53,7 +53,6 @@ "minimist": "^1.2.6", "p-limit": "^3.1.0", "path2": "^0.1.0", - "promise-queue": "^2.2.5", "ramda": "^0.28.0", "semver": "^7.3.7", "signal-exit": "^3.0.7", @@ -61,8 +60,7 @@ "supports-color": "^8.1.1", "traverse": "^0.6.6", "type": "^2.6.1", - "uni-global": "^1.0.0", - "uuid": "^8.3.2" + "uni-global": "^1.0.0" }, "devDependencies": { "@types/mocha": "^9.1.1", diff --git a/src/state/S3StateStorage.js b/src/state/S3StateStorage.js index afd32b4..aef56b7 100644 --- a/src/state/S3StateStorage.js +++ b/src/state/S3StateStorage.js @@ -1,13 +1,11 @@ 'use strict'; const { S3 } = require('@aws-sdk/client-s3'); -const PromiseQueue = require('promise-queue'); +const pLimit = require('p-limit'); const streamToString = require('../utils/stream-to-string'); const ServerlessError = require('../serverless-error'); const BaseStateStorage = require('./BaseStateStorage'); -PromiseQueue.configure(Promise); - class S3StateStorage extends BaseStateStorage { constructor(config = {}) { super(); @@ -20,7 +18,7 @@ class S3StateStorage extends BaseStateStorage { this.s3Client = new S3({ region: this.region, credentials: config.credentials }); - this.writeRequestQueue = new PromiseQueue(1, Infinity); + this.writeRequestQueue = pLimit(1); } async readState() { @@ -52,7 +50,7 @@ class S3StateStorage extends BaseStateStorage { async writeState() { try { - await this.writeRequestQueue.add(async () => { + await this.writeRequestQueue(async () => { await this.s3Client.putObject({ Bucket: this.bucketName, Key: this.stateKey, diff --git a/src/utils/glob.js b/src/utils/glob.js new file mode 100644 index 0000000..cc2568c --- /dev/null +++ b/src/utils/glob.js @@ -0,0 +1,165 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const fastGlob = require('fast-glob'); + +const toArray = (patterns) => { + return Array.isArray(patterns) ? patterns : [patterns]; +}; + +const isDynamicPattern = (pattern) => /[*?{}()[\]]/.test(pattern); + +const maybeExpandDirectoryPattern = (pattern, cwd) => { + if (isDynamicPattern(pattern)) { + return pattern; + } + + const absolutePattern = path.resolve(cwd || process.cwd(), pattern); + try { + if (fs.statSync(absolutePattern).isDirectory()) { + const normalizedPattern = pattern.replace(/\\/g, '/').replace(/\/$/, ''); + return `${normalizedPattern}/**/*`; + } + } catch { + // Ignore missing paths and let fast-glob handle them. + } + + return pattern; +}; + +const expandPattern = (pattern, cwd, expandDirectories) => { + if (!expandDirectories) { + return pattern; + } + + if (pattern.startsWith('!')) { + return `!${maybeExpandDirectoryPattern(pattern.slice(1), cwd)}`; + } + + return maybeExpandDirectoryPattern(pattern, cwd); +}; + +const normalizeOptions = (options = {}) => { + const globOptions = { ...options }; + const expandDirectories = + globOptions.expandDirectories !== undefined ? globOptions.expandDirectories : true; + + delete globOptions.expandDirectories; + delete globOptions.nosort; + + if (globOptions.follow !== undefined) { + globOptions.followSymbolicLinks = globOptions.follow; + delete globOptions.follow; + } + + if (globOptions.nodir !== undefined) { + globOptions.onlyFiles = globOptions.nodir; + delete globOptions.nodir; + } + + if (globOptions.silent !== undefined) { + globOptions.suppressErrors = globOptions.silent; + delete globOptions.silent; + } + + return { expandDirectories, globOptions }; +}; + +const buildTasks = (patterns, globOptions) => { + const tasks = []; + let remainingPatterns = patterns; + + while (remainingPatterns.length > 0) { + const negativePatternIndex = remainingPatterns.findIndex((pattern) => pattern.startsWith('!')); + + if (negativePatternIndex === -1) { + tasks.push({ + patterns: remainingPatterns, + options: { + ...globOptions, + ignore: [...(globOptions.ignore || [])], + }, + }); + break; + } + + const ignorePattern = remainingPatterns[negativePatternIndex].slice(1); + + for (const task of tasks) { + task.options.ignore.push(ignorePattern); + } + + if (negativePatternIndex !== 0) { + tasks.push({ + patterns: remainingPatterns.slice(0, negativePatternIndex), + options: { + ...globOptions, + ignore: [...(globOptions.ignore || []), ignorePattern], + }, + }); + } + + remainingPatterns = remainingPatterns.slice(negativePatternIndex + 1); + } + + return tasks; +}; + +const collectResults = async (tasks) => { + const seen = new Set(); + const results = []; + + for (const task of tasks) { + for (const entry of await fastGlob(task.patterns, task.options)) { + if (seen.has(entry)) { + continue; + } + + seen.add(entry); + results.push(entry); + } + } + + return results; +}; + +const collectResultsSync = (tasks) => { + const seen = new Set(); + const results = []; + + for (const task of tasks) { + for (const entry of fastGlob.sync(task.patterns, task.options)) { + if (seen.has(entry)) { + continue; + } + + seen.add(entry); + results.push(entry); + } + } + + return results; +}; + +const glob = async (patterns, options = {}) => { + const normalizedPatterns = toArray(patterns); + const { expandDirectories, globOptions } = normalizeOptions(options); + const expandedPatterns = normalizedPatterns.map((pattern) => + expandPattern(pattern, globOptions.cwd, expandDirectories) + ); + + return collectResults(buildTasks(expandedPatterns, globOptions)); +}; + +glob.sync = (patterns, options = {}) => { + const normalizedPatterns = toArray(patterns); + const { expandDirectories, globOptions } = normalizeOptions(options); + const expandedPatterns = normalizedPatterns.map((pattern) => + expandPattern(pattern, globOptions.cwd, expandDirectories) + ); + + return collectResultsSync(buildTasks(expandedPatterns, globOptions)); +}; + +module.exports = glob; diff --git a/test/unit/components/framework/index.test.js b/test/unit/components/framework/index.test.js index 716c0ce..7a8c4b0 100644 --- a/test/unit/components/framework/index.test.js +++ b/test/unit/components/framework/index.test.js @@ -1,7 +1,10 @@ 'use strict'; +const fs = require('node:fs').promises; +const path = require('path'); const proxyquire = require('proxyquire'); const chai = require('chai'); +const fse = require('fs-extra'); const sinon = require('sinon'); const Context = require('../../../../src/Context'); const ComponentContext = require('../../../../src/ComponentContext'); @@ -545,4 +548,56 @@ describe('test/unit/components/framework/index.test.js', () => { .to.throw() .and.have.property('code', 'INVALID_PATH_IN_SERVICE_CONFIGURATION'); }); + + it('expands literal directory cache patterns when calculating hashes', async () => { + const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-dir-')); + + try { + await fse.outputFile(path.join(serviceDir, 'src', 'handler.js'), 'module.exports = 1;\n'); + + const context = await getContext(); + const directoryPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['src'], + }); + const globPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['src/**/*'], + }); + + expect(await directoryPatternComponent.calculateCacheHash()).to.equal( + await globPatternComponent.calculateCacheHash() + ); + } finally { + await fse.remove(serviceDir); + } + }); + + it('supports negated cache patterns that re-include a later file', async () => { + const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-negation-')); + + try { + await Promise.all([ + fse.outputFile(path.join(serviceDir, 'keep.js'), 'keep\n'), + fse.outputFile(path.join(serviceDir, 'ignored', 'drop.js'), 'drop\n'), + fse.outputFile(path.join(serviceDir, 'ignored', 'reinclude.js'), 'reinclude\n'), + ]); + + const context = await getContext(); + const negatedPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['**/*', '!ignored/**/*', 'ignored/reinclude.js'], + }); + const explicitPatternComponent = new ServerlessFramework('id', context, { + path: serviceDir, + cachePatterns: ['keep.js', 'ignored/reinclude.js'], + }); + + expect(await negatedPatternComponent.calculateCacheHash()).to.equal( + await explicitPatternComponent.calculateCacheHash() + ); + } finally { + await fse.remove(serviceDir); + } + }); }); diff --git a/test/unit/src/state/S3StateStorage.test.js b/test/unit/src/state/S3StateStorage.test.js index 3dfd721..ce67b72 100644 --- a/test/unit/src/state/S3StateStorage.test.js +++ b/test/unit/src/state/S3StateStorage.test.js @@ -9,6 +9,17 @@ const S3StateStorage = require('../../../../src/state/S3StateStorage'); const expect = chai.expect; +const createDeferred = () => { + let resolve; + let reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + + return { promise, resolve, reject }; +}; + describe('test/unit/src/state/S3StateStorage.test.js', () => { const bucketName = 'dummy-bucket'; const stateKey = 'dummy-key'; @@ -108,4 +119,45 @@ describe('test/unit/src/state/S3StateStorage.test.js', () => { credentials: 'creds', }); }); + + it('serializes concurrent state writes', async () => { + const firstWrite = createDeferred(); + const secondWrite = createDeferred(); + let activeWrites = 0; + let maxActiveWrites = 0; + const s3StateStorage = new S3StateStorage({ bucketName, stateKey }); + const mockedS3Client = { + putObject: sinon.stub().callsFake(() => { + activeWrites += 1; + maxActiveWrites = Math.max(maxActiveWrites, activeWrites); + const currentWrite = mockedS3Client.putObject.callCount === 1 ? firstWrite : secondWrite; + return currentWrite.promise.finally(() => { + activeWrites -= 1; + }); + }), + }; + + s3StateStorage.s3Client = mockedS3Client; + s3StateStorage.state = { first: true }; + + const firstPromise = s3StateStorage.writeState(); + s3StateStorage.state = { second: true }; + const secondPromise = s3StateStorage.writeState(); + + await Promise.resolve(); + await Promise.resolve(); + + expect(mockedS3Client.putObject.calledOnce).to.equal(true); + + firstWrite.resolve(); + await firstPromise; + await Promise.resolve(); + + expect(mockedS3Client.putObject.calledTwice).to.equal(true); + + secondWrite.resolve(); + await Promise.all([firstPromise, secondPromise]); + + expect(maxActiveWrites).to.equal(1); + }); }); diff --git a/test/unit/src/utils/glob.test.js b/test/unit/src/utils/glob.test.js new file mode 100644 index 0000000..dafd1c3 --- /dev/null +++ b/test/unit/src/utils/glob.test.js @@ -0,0 +1,52 @@ +'use strict'; + +const fs = require('node:fs').promises; +const os = require('node:os'); +const path = require('node:path'); +const fse = require('fs-extra'); +const { expect } = require('chai'); + +const glob = require('../../../../src/utils/glob'); + +describe('test/unit/src/utils/glob.test.js', () => { + let tmpDir; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'compose-glob-')); + }); + + afterEach(async () => { + await fse.remove(tmpDir); + }); + + it('matches globby order-sensitive leading negation behavior for async and sync calls', async () => { + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'keep.js'), 'keep\n'), + fse.outputFile(path.join(tmpDir, 'keep.ts'), 'keep\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'drop.js'), 'drop\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'drop.ts'), 'drop\n'), + ]); + + expect( + (await glob(['!ignored/**/*', '**/*.js', '**/*.ts'], { cwd: tmpDir })).sort() + ).to.deep.equal(['ignored/drop.js', 'ignored/drop.ts', 'keep.js', 'keep.ts']); + expect( + glob.sync(['!ignored/**/*', '**/*.js', '**/*.ts'], { cwd: tmpDir }).sort() + ).to.deep.equal(['ignored/drop.js', 'ignored/drop.ts', 'keep.js', 'keep.ts']); + }); + + it('applies later negations to earlier positives and still supports re-inclusion', async () => { + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'keep.js'), 'keep\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'drop.js'), 'drop\n'), + fse.outputFile(path.join(tmpDir, 'ignored', 'reinclude.js'), 'reinclude\n'), + ]); + + expect( + (await glob(['**/*.js', '!ignored/**/*', 'ignored/reinclude.js'], { cwd: tmpDir })).sort() + ).to.deep.equal(['ignored/reinclude.js', 'keep.js']); + expect( + glob.sync(['**/*.js', '!ignored/**/*', 'ignored/reinclude.js'], { cwd: tmpDir }).sort() + ).to.deep.equal(['ignored/reinclude.js', 'keep.js']); + }); +});