diff --git a/components/framework/index.js b/components/framework/index.js index a17dac6..68454ca 100644 --- a/components/framework/index.js +++ b/components/framework/index.js @@ -1,11 +1,10 @@ 'use strict'; const YAML = require('js-yaml'); -const hasha = require('hasha'); -const glob = require('../../src/utils/glob'); const path = require('path'); const spawn = require('../../src/utils/spawn'); const semver = require('semver'); +const calculateCacheHash = require('../../src/utils/cache-hash'); const { configSchema } = require('./configuration'); const ServerlessError = require('../../src/serverless-error'); @@ -350,22 +349,7 @@ class ServerlessFramework { * @return {Promise} */ async calculateCacheHash() { - const algorithm = 'md5'; // fastest - - const allFilePaths = await glob(this.inputs.cachePatterns, { - cwd: this.inputs.path, - }); - - const promises = []; - for (const filePath of allFilePaths) { - promises.push(hasha.fromFile(path.join(this.inputs.path, filePath), { algorithm })); - } - const hashes = await Promise.all(promises); - - // Sort hashes to avoid having the final hash change just because files where read in a different order - hashes.sort(); - - return hasha(hashes.join(), { algorithm }); + return calculateCacheHash(this.inputs.cachePatterns, this.inputs.path); } } diff --git a/package.json b/package.json index d6d3ea4..26b3128 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,6 @@ "ext": "^1.7.0", "fast-glob": "^3.3.3", "fs-extra": "^10.1.0", - "hasha": "^5.2.2", "is-interactive": "^1", "is-unicode-supported": "^0.1", "js-yaml": "^4.1.0", diff --git a/src/utils/cache-hash.js b/src/utils/cache-hash.js new file mode 100644 index 0000000..78e8c96 --- /dev/null +++ b/src/utils/cache-hash.js @@ -0,0 +1,36 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const glob = require('./glob'); + +const CACHE_HASH_ALGORITHM = 'md5'; +const CACHE_HASH_ENCODING = 'hex'; + +const createCacheHash = () => crypto.createHash(CACHE_HASH_ALGORITHM); + +const calculateCacheDigest = (input) => createCacheHash().update(input).digest(CACHE_HASH_ENCODING); + +const calculateFileCacheDigest = (filePath) => + new Promise((resolve, reject) => { + const hash = createCacheHash(); + const stream = fs.createReadStream(filePath); + + stream.on('error', reject); + stream.on('data', (chunk) => hash.update(chunk)); + stream.on('end', () => resolve(hash.digest(CACHE_HASH_ENCODING))); + }); + +const calculateCacheHash = async (patterns, cwd) => { + const allFilePaths = await glob(patterns, { cwd }); + const hashes = await Promise.all( + allFilePaths.map((filePath) => calculateFileCacheDigest(path.join(cwd, filePath))) + ); + + hashes.sort(); + + return calculateCacheDigest(hashes.join()); +}; + +module.exports = calculateCacheHash; diff --git a/test/unit/components/framework/index.test.js b/test/unit/components/framework/index.test.js index 7ad9a5c..dfc3504 100644 --- a/test/unit/components/framework/index.test.js +++ b/test/unit/components/framework/index.test.js @@ -832,6 +832,76 @@ describe('test/unit/components/framework/index.test.js', () => { .and.have.property('code', 'INVALID_PATH_IN_SERVICE_CONFIGURATION'); }); + it('skips deploy when cache inputs and cache hash are unchanged', async () => { + const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-skip-')); + + try { + await fse.outputFile(path.join(serviceDir, 'handler.js'), 'module.exports = 1;\n'); + + const spawnStub = sinon.stub(); + const FrameworkComponent = proxyquire('../../../../components/framework/index.js', { + '../../src/utils/spawn': spawnStub, + }); + + const context = await getContext(); + const inputs = { + path: serviceDir, + cachePatterns: ['handler.js'], + }; + const component = new FrameworkComponent('id', context, inputs); + + context.state.inputs = inputs; + context.state.cacheHash = await component.calculateCacheHash(); + + await component.deploy(); + + expect(spawnStub).to.not.have.been.called; + } finally { + await fse.remove(serviceDir); + } + }); + + it('updates cache hash after deploying changed cache pattern files', async () => { + const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-update-')); + + try { + const filePath = path.join(serviceDir, 'handler.js'); + await fse.outputFile(filePath, 'module.exports = 1;\n'); + + const spawnStub = sinon.stub(); + spawnStub.onFirstCall().returns(createSpawnExecution({ stderr: 'deployed' })); + spawnStub.onSecondCall().returns( + createSpawnExecution({ + stdout: 'region: us-east-1\n\nStack Outputs:\n Key: Output', + }) + ); + + const FrameworkComponent = proxyquire('../../../../components/framework/index.js', { + '../../src/utils/spawn': spawnStub, + }); + + const context = await getContext(); + const inputs = { + path: serviceDir, + cachePatterns: ['handler.js'], + }; + const component = new FrameworkComponent('id', context, inputs); + + context.state.detectedFrameworkVersion = '9.9.9'; + context.state.inputs = inputs; + context.state.cacheHash = await component.calculateCacheHash(); + + await fse.outputFile(filePath, 'module.exports = 2;\n'); + + await component.deploy(); + + expect(spawnStub).to.be.calledTwice; + expect(context.state.cacheHash).to.equal(await component.calculateCacheHash()); + } finally { + await fse.remove(serviceDir); + } + }); + it('expands literal directory cache patterns when calculating hashes', async () => { const serviceDir = await fs.mkdtemp(path.join(process.cwd(), 'cache-hash-dir-')); diff --git a/test/unit/src/utils/cache-hash.test.js b/test/unit/src/utils/cache-hash.test.js new file mode 100644 index 0000000..28d1ac3 --- /dev/null +++ b/test/unit/src/utils/cache-hash.test.js @@ -0,0 +1,148 @@ +'use strict'; + +const fs = require('node:fs').promises; +const crypto = require('node:crypto'); +const os = require('node:os'); +const path = require('node:path'); +const proxyquire = require('proxyquire'); +const fse = require('fs-extra'); +const { expect } = require('chai'); +const sinon = require('sinon'); + +const calculateCacheHash = require('../../../../src/utils/cache-hash'); + +const md5Hex = (input) => crypto.createHash('md5').update(input).digest('hex'); + +const expectedCacheHashForContents = (contents) => + md5Hex( + contents + .map((content) => md5Hex(Buffer.isBuffer(content) ? content : Buffer.from(content))) + .sort() + .join() + ); + +describe('test/unit/src/utils/cache-hash.test.js', () => { + let tmpDir; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'compose-cache-hash-')); + }); + + afterEach(async () => { + await fse.remove(tmpDir); + }); + + it('calculates the md5 cache hash contract for a single file', async () => { + const contents = 'module.exports = 1;\n'; + await fse.outputFile(path.join(tmpDir, 'handler.js'), contents); + + expect(await calculateCacheHash(['handler.js'], tmpDir)).to.equal( + md5Hex(md5Hex(Buffer.from(contents))) + ); + }); + + it('calculates cache hashes independently of cache pattern order', async () => { + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'a.txt'), 'aaa\n'), + fse.outputFile(path.join(tmpDir, 'b.txt'), 'bbb\n'), + fse.outputFile(path.join(tmpDir, 'c.txt'), 'ccc\n'), + ]); + + const expected = expectedCacheHashForContents(['aaa\n', 'bbb\n', 'ccc\n']); + + expect(await calculateCacheHash(['a.txt', 'b.txt', 'c.txt'], tmpDir)).to.equal(expected); + expect(await calculateCacheHash(['c.txt', 'a.txt', 'b.txt'], tmpDir)).to.equal(expected); + }); + + it('sorts cache file hashes instead of file paths', async () => { + const contentsByHash = ['first\n', 'second\n'] + .map((contents) => ({ contents, hash: md5Hex(Buffer.from(contents)) })) + .sort((a, b) => a.hash.localeCompare(b.hash)); + const [lowerHashEntry, higherHashEntry] = contentsByHash; + + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'a.txt'), higherHashEntry.contents), + fse.outputFile(path.join(tmpDir, 'z.txt'), lowerHashEntry.contents), + ]); + + const expected = md5Hex([lowerHashEntry.hash, higherHashEntry.hash].join()); + const pathOrderedHash = md5Hex([higherHashEntry.hash, lowerHashEntry.hash].join()); + + expect(expected).to.not.equal(pathOrderedHash); + expect(await calculateCacheHash(['*.txt'], tmpDir)).to.equal(expected); + }); + + it('counts duplicate file contents when calculating cache hashes', async () => { + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'one.txt'), 'same\n'), + fse.outputFile(path.join(tmpDir, 'two.txt'), 'same\n'), + ]); + + const fileHash = md5Hex(Buffer.from('same\n')); + const expected = md5Hex([fileHash, fileHash].sort().join()); + const actual = await calculateCacheHash(['*.txt'], tmpDir); + + expect(actual).to.equal(expected); + expect(actual).to.not.equal(md5Hex(fileHash)); + }); + + it('does not include file paths in cache hashes', async () => { + const otherDir = await fs.mkdtemp(path.join(os.tmpdir(), 'compose-cache-hash-other-')); + + try { + await fse.outputFile(path.join(tmpDir, 'one.txt'), 'content\n'); + await fse.outputFile(path.join(otherDir, 'nested', 'different-name.js'), 'content\n'); + + expect(await calculateCacheHash(['one.txt'], tmpDir)).to.equal( + await calculateCacheHash(['nested/different-name.js'], otherDir) + ); + } finally { + await fse.remove(otherDir); + } + }); + + it('hashes binary cache pattern files as raw bytes', async () => { + const contents = Buffer.from([0x00, 0xff, 0x80, 0x0a]); + await fse.outputFile(path.join(tmpDir, 'binary.bin'), contents); + + expect(await calculateCacheHash(['binary.bin'], tmpDir)).to.equal( + expectedCacheHashForContents([contents]) + ); + }); + + it('returns the md5 of an empty string when no cache pattern files match', async () => { + expect(await calculateCacheHash(['does-not-exist/**/*'], tmpDir)).to.equal( + 'd41d8cd98f00b204e9800998ecf8427e' + ); + }); + + it('includes empty files in cache hashes', async () => { + await fse.outputFile(path.join(tmpDir, 'empty.txt'), ''); + + const actual = await calculateCacheHash(['empty.txt'], tmpDir); + + expect(actual).to.equal(expectedCacheHashForContents([Buffer.alloc(0)])); + expect(actual).to.not.equal('d41d8cd98f00b204e9800998ecf8427e'); + }); + + it('uses comma-separated file hashes for the aggregate cache hash', async () => { + await Promise.all([ + fse.outputFile(path.join(tmpDir, 'a.txt'), 'a'), + fse.outputFile(path.join(tmpDir, 'b.txt'), 'b'), + ]); + + const hashes = [md5Hex(Buffer.from('a')), md5Hex(Buffer.from('b'))].sort(); + const actual = await calculateCacheHash(['*.txt'], tmpDir); + + expect(actual).to.equal(md5Hex(hashes.join())); + expect(actual).to.not.equal(md5Hex(hashes.join(''))); + }); + + it('rejects when a matched cache pattern file cannot be read', async () => { + const calculateCacheHashWithMissingFile = proxyquire('../../../../src/utils/cache-hash', { + './glob': sinon.stub().resolves(['missing.txt']), + }); + + await expect(calculateCacheHashWithMissingFile(['**/*'], tmpDir)).to.be.rejected; + }); +});