Replace various old packages

components/framework/index.js

@@ -3,7 +3,7 @@
 const spawn = require('cross-spawn');
 const YAML = require('js-yaml');
 const hasha = require('hasha');
-const globby = require('globby');
+const glob = require('../../src/utils/glob');
 const path = require('path');
 const spawnExt = require('child-process-ext/spawn');
 const semver = require('semver');
@@ -316,7 +316,7 @@ class ServerlessFramework {
   async calculateCacheHash() {
     const algorithm = 'md5'; // fastest
 
-    const allFilePaths = await globby(this.inputs.cachePatterns, {
+    const allFilePaths = await glob(this.inputs.cachePatterns, {
       cwd: this.inputs.path,
     });
 

package.json

@@ -35,14 +35,14 @@
     "chalk": "^4.1.2",
     "child-process-ext": "^2.1.1",
     "ci-info": "^3.3.2",
-    "cli-progress-footer": "^2.3.2",
     "cli-cursor": "^3",
-    "d": "^1.0.1",
+    "cli-progress-footer": "^2.3.2",
     "cross-spawn": "^7.0.3",
+    "d": "^1.0.1",
     "event-emitter": "^0.3.5",
     "ext": "^1.7.0",
+    "fast-glob": "^3.3.3",
     "fs-extra": "^10.1.0",
-    "globby": "^11.1.0",
     "hasha": "^5.2.2",
     "is-interactive": "^1",
     "is-unicode-supported": "^0.1",
@@ -53,16 +53,14 @@
     "minimist": "^1.2.6",
     "p-limit": "^3.1.0",
     "path2": "^0.1.0",
-    "promise-queue": "^2.2.5",
     "ramda": "^0.28.0",
     "semver": "^7.3.7",
     "signal-exit": "^3.0.7",
     "strip-ansi": "^6.0.1",
     "supports-color": "^8.1.1",
     "traverse": "^0.6.6",
     "type": "^2.6.1",
-    "uni-global": "^1.0.0",
-    "uuid": "^8.3.2"
+    "uni-global": "^1.0.0"
   },
   "devDependencies": {
     "@types/mocha": "^9.1.1",

src/state/S3StateStorage.js

@@ -1,13 +1,11 @@
 'use strict';
 
 const { S3 } = require('@aws-sdk/client-s3');
-const PromiseQueue = require('promise-queue');
+const pLimit = require('p-limit');
 const streamToString = require('../utils/stream-to-string');
 const ServerlessError = require('../serverless-error');
 const BaseStateStorage = require('./BaseStateStorage');
 
-PromiseQueue.configure(Promise);
-
 class S3StateStorage extends BaseStateStorage {
   constructor(config = {}) {
     super();
@@ -20,7 +18,7 @@ class S3StateStorage extends BaseStateStorage {
 
     this.s3Client = new S3({ region: this.region, credentials: config.credentials });
 
-    this.writeRequestQueue = new PromiseQueue(1, Infinity);
+    this.writeRequestQueue = pLimit(1);
   }
 
   async readState() {
@@ -52,7 +50,7 @@ class S3StateStorage extends BaseStateStorage {
 
   async writeState() {
     try {
-      await this.writeRequestQueue.add(async () => {
+      await this.writeRequestQueue(async () => {
         await this.s3Client.putObject({
           Bucket: this.bucketName,
           Key: this.stateKey,

src/utils/glob.js

@@ -0,0 +1,165 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const fastGlob = require('fast-glob');
+
+const toArray = (patterns) => {
+  return Array.isArray(patterns) ? patterns : [patterns];
+};
+
+const isDynamicPattern = (pattern) => /[*?{}()[\]]/.test(pattern);
+
+const maybeExpandDirectoryPattern = (pattern, cwd) => {
+  if (isDynamicPattern(pattern)) {
+    return pattern;
+  }
+
+  const absolutePattern = path.resolve(cwd || process.cwd(), pattern);
+  try {
+    if (fs.statSync(absolutePattern).isDirectory()) {
+      const normalizedPattern = pattern.replace(/\\/g, '/').replace(/\/$/, '');
+      return `${normalizedPattern}/**/*`;
+    }
+  } catch {
+    // Ignore missing paths and let fast-glob handle them.
+  }
+
+  return pattern;
+};
+
+const expandPattern = (pattern, cwd, expandDirectories) => {
+  if (!expandDirectories) {
+    return pattern;
+  }
+
+  if (pattern.startsWith('!')) {
+    return `!${maybeExpandDirectoryPattern(pattern.slice(1), cwd)}`;
+  }
+
+  return maybeExpandDirectoryPattern(pattern, cwd);
+};
+
+const normalizeOptions = (options = {}) => {
+  const globOptions = { ...options };
+  const expandDirectories =
+    globOptions.expandDirectories !== undefined ? globOptions.expandDirectories : true;
+
+  delete globOptions.expandDirectories;
+  delete globOptions.nosort;
+
+  if (globOptions.follow !== undefined) {
+    globOptions.followSymbolicLinks = globOptions.follow;
+    delete globOptions.follow;
+  }
+
+  if (globOptions.nodir !== undefined) {
+    globOptions.onlyFiles = globOptions.nodir;
+    delete globOptions.nodir;
+  }
+
+  if (globOptions.silent !== undefined) {
+    globOptions.suppressErrors = globOptions.silent;
+    delete globOptions.silent;
+  }
+
+  return { expandDirectories, globOptions };
+};
+
+const buildTasks = (patterns, globOptions) => {
+  const tasks = [];
+  let remainingPatterns = patterns;
+
+  while (remainingPatterns.length > 0) {
+    const negativePatternIndex = remainingPatterns.findIndex((pattern) => pattern.startsWith('!'));
+
+    if (negativePatternIndex === -1) {
+      tasks.push({
+        patterns: remainingPatterns,
+        options: {
+          ...globOptions,
+          ignore: [...(globOptions.ignore || [])],
+        },
+      });
+      break;
+    }
+
+    const ignorePattern = remainingPatterns[negativePatternIndex].slice(1);
+
+    for (const task of tasks) {
+      task.options.ignore.push(ignorePattern);
+    }
+
+    if (negativePatternIndex !== 0) {
+      tasks.push({
+        patterns: remainingPatterns.slice(0, negativePatternIndex),
+        options: {
+          ...globOptions,
+          ignore: [...(globOptions.ignore || []), ignorePattern],
+        },
+      });
+    }
+
+    remainingPatterns = remainingPatterns.slice(negativePatternIndex + 1);
+  }
+
+  return tasks;
+};
+
+const collectResults = async (tasks) => {
+  const seen = new Set();
+  const results = [];
+
+  for (const task of tasks) {
+    for (const entry of await fastGlob(task.patterns, task.options)) {
+      if (seen.has(entry)) {
+        continue;
+      }
+
+      seen.add(entry);
+      results.push(entry);
+    }
+  }
+
+  return results;
+};
+
+const collectResultsSync = (tasks) => {
+  const seen = new Set();
+  const results = [];
+
+  for (const task of tasks) {
+    for (const entry of fastGlob.sync(task.patterns, task.options)) {
+      if (seen.has(entry)) {
+        continue;
+      }
+
+      seen.add(entry);
+      results.push(entry);
+    }
+  }
+
+  return results;
+};
+
+const glob = async (patterns, options = {}) => {
+  const normalizedPatterns = toArray(patterns);
+  const { expandDirectories, globOptions } = normalizeOptions(options);
+  const expandedPatterns = normalizedPatterns.map((pattern) =>
+    expandPattern(pattern, globOptions.cwd, expandDirectories)
+  );
+
+  return collectResults(buildTasks(expandedPatterns, globOptions));
+};
+
+glob.sync = (patterns, options = {}) => {
+  const normalizedPatterns = toArray(patterns);
+  const { expandDirectories, globOptions } = normalizeOptions(options);
+  const expandedPatterns = normalizedPatterns.map((pattern) =>
+    expandPattern(pattern, globOptions.cwd, expandDirectories)
+  );
+
+  return collectResultsSync(buildTasks(expandedPatterns, globOptions));
+};
+
+module.exports = glob;

test/unit/components/framework/index.test.js

@@ -1,7 +1,10 @@
 'use strict';
 
+const fs = require('node:fs').promises;
+const path = require('path');
 const proxyquire = require('proxyquire');
 const chai = require('chai');
+const fse = require('fs-extra');
 const sinon = require('sinon');
 const Context = require('../../../../src/Context');
 const ComponentContext = require('../../../../src/ComponentContext');
@@ -545,4 +548,56 @@ describe('test/unit/components/framework/index.test.js', () => {
       .to.throw()
       .and.have.property('code', 'INVALID_PATH_IN_SERVICE_CONFIGURATION');
   });
+
+  it('expands literal directory cache patterns when calculating hashes', async () => {
+    const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-dir-'));
+
+    try {
+      await fse.outputFile(path.join(serviceDir, 'src', 'handler.js'), 'module.exports = 1;\n');
+
+      const context = await getContext();
+      const directoryPatternComponent = new ServerlessFramework('id', context, {
+        path: serviceDir,
+        cachePatterns: ['src'],
+      });
+      const globPatternComponent = new ServerlessFramework('id', context, {
+        path: serviceDir,
+        cachePatterns: ['src/**/*'],
+      });
+
+      expect(await directoryPatternComponent.calculateCacheHash()).to.equal(
+        await globPatternComponent.calculateCacheHash()
+      );
+    } finally {
+      await fse.remove(serviceDir);
+    }
+  });
+
+  it('supports negated cache patterns that re-include a later file', async () => {
+    const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-negation-'));
+
+    try {
+      await Promise.all([
+        fse.outputFile(path.join(serviceDir, 'keep.js'), 'keep\n'),
+        fse.outputFile(path.join(serviceDir, 'ignored', 'drop.js'), 'drop\n'),
+        fse.outputFile(path.join(serviceDir, 'ignored', 'reinclude.js'), 'reinclude\n'),
+      ]);
+
+      const context = await getContext();
+      const negatedPatternComponent = new ServerlessFramework('id', context, {
+        path: serviceDir,
+        cachePatterns: ['**/*', '!ignored/**/*', 'ignored/reinclude.js'],
+      });
+      const explicitPatternComponent = new ServerlessFramework('id', context, {
+        path: serviceDir,
+        cachePatterns: ['keep.js', 'ignored/reinclude.js'],
+      });
+
+      expect(await negatedPatternComponent.calculateCacheHash()).to.equal(
+        await explicitPatternComponent.calculateCacheHash()
+      );
+    } finally {
+      await fse.remove(serviceDir);
+    }
+  });
 });

test/unit/src/state/S3StateStorage.test.js

@@ -9,6 +9,17 @@ const S3StateStorage = require('../../../../src/state/S3StateStorage');
 
 const expect = chai.expect;
 
+const createDeferred = () => {
+  let resolve;
+  let reject;
+  const promise = new Promise((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+
+  return { promise, resolve, reject };
+};
+
 describe('test/unit/src/state/S3StateStorage.test.js', () => {
   const bucketName = 'dummy-bucket';
   const stateKey = 'dummy-key';
@@ -108,4 +119,45 @@ describe('test/unit/src/state/S3StateStorage.test.js', () => {
       credentials: 'creds',
     });
   });
+
+  it('serializes concurrent state writes', async () => {
+    const firstWrite = createDeferred();
+    const secondWrite = createDeferred();
+    let activeWrites = 0;
+    let maxActiveWrites = 0;
+    const s3StateStorage = new S3StateStorage({ bucketName, stateKey });
+    const mockedS3Client = {
+      putObject: sinon.stub().callsFake(() => {
+        activeWrites += 1;
+        maxActiveWrites = Math.max(maxActiveWrites, activeWrites);
+        const currentWrite = mockedS3Client.putObject.callCount === 1 ? firstWrite : secondWrite;
+        return currentWrite.promise.finally(() => {
+          activeWrites -= 1;
+        });
+      }),
+    };
+
+    s3StateStorage.s3Client = mockedS3Client;
+    s3StateStorage.state = { first: true };
+
+    const firstPromise = s3StateStorage.writeState();
+    s3StateStorage.state = { second: true };
+    const secondPromise = s3StateStorage.writeState();
+
+    await Promise.resolve();
+    await Promise.resolve();
+
+    expect(mockedS3Client.putObject.calledOnce).to.equal(true);
+
+    firstWrite.resolve();
+    await firstPromise;
+    await Promise.resolve();
+
+    expect(mockedS3Client.putObject.calledTwice).to.equal(true);
+
+    secondWrite.resolve();
+    await Promise.all([firstPromise, secondPromise]);
+
+    expect(maxActiveWrites).to.equal(1);
+  });
 });