crypto: validate digest output encodings

Reject unknown Hash and Hmac digest encodings before finalizing the
operation instead of silently returning a Buffer. Preserve buffer
output aliases and existing encoding coercion.

Signed-off-by: Filip Skokan <panva.ip@gmail.com>
Assisted-by: Codex
PR-URL: #66247
Fixes: #45189
Refs: #45990
Reviewed-By: Xuguang Mei <meixuguang@gmail.com>
Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: panva

lib/internal/crypto/hash.js

@@ -52,6 +52,7 @@ const {
     ERR_CRYPTO_HASH_UPDATE_FAILED,
     ERR_INVALID_ARG_TYPE,
     ERR_INVALID_ARG_VALUE,
+    ERR_UNKNOWN_ENCODING,
   },
 } = require('internal/errors');
 
@@ -176,13 +177,23 @@ Hash.prototype.update = function update(data, encoding) {
 };
 
 
+function validateDigestEncoding(outputEncoding) {
+  // Explicit conversion of truthy values for backward compatibility.
+  outputEncoding &&= `${outputEncoding}`;
+  if (outputEncoding &&
+      normalizeEncoding(outputEncoding) === undefined &&
+      StringPrototypeToLowerCase(outputEncoding) !== 'buffer') {
+    throw new ERR_UNKNOWN_ENCODING(outputEncoding);
+  }
+  return outputEncoding;
+}
+
 Hash.prototype.digest = function digest(outputEncoding) {
   const state = this[kState];
   if (state[kFinalized])
     throw new ERR_CRYPTO_HASH_FINALIZED();
 
-  // Explicit conversion of truthy values for backward compatibility.
-  const ret = this[kHandle].digest(outputEncoding && `${outputEncoding}`);
+  const ret = this[kHandle].digest(validateDigestEncoding(outputEncoding));
   state[kFinalized] = true;
   return ret;
 };
@@ -217,8 +228,7 @@ Hmac.prototype.digest = function digest(outputEncoding) {
     return buf;
   }
 
-  // Explicit conversion of truthy values for backward compatibility.
-  const ret = this[kHandle].digest(outputEncoding && `${outputEncoding}`);
+  const ret = this[kHandle].digest(validateDigestEncoding(outputEncoding));
   state[kFinalized] = true;
   return ret;
 };

test/parallel/test-crypto-digest-output-encoding.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const common = require('../common');
+if (!common.hasCrypto)
+  common.skip('missing crypto');
+
+const assert = require('assert');
+const crypto = require('crypto');
+
+for (const create of [
+  () => crypto.createHash('sha256'),
+  () => crypto.createHmac('sha256', 'key'),
+]) {
+  const expected = create().update('test').digest();
+  for (const encoding of ['bad', { toString: () => 'bad' }]) {
+    const instance = create().update('test');
+    assert.throws(() => instance.digest(encoding), {
+      code: 'ERR_UNKNOWN_ENCODING',
+      message: 'Unknown encoding: bad',
+    });
+    // An invalid encoding must not finalize the operation.
+    assert.deepStrictEqual(instance.digest(), expected);
+  }
+  for (const encoding of [undefined, null, '', false, 0, 'buffer', 'BUFFER']) {
+    assert.deepStrictEqual(create().update('test').digest(encoding), expected);
+  }
+  for (const encoding of ['hex', 'HEX', 'base64', 'base64url', 'latin1', 'utf-8']) {
+    assert.strictEqual(create().update('test').digest(encoding),
+                       expected.toString(encoding));
+  }
+  assert.strictEqual(
+    create().update('test').digest({ toString: () => 'hex' }),
+    expected.toString('hex'));
+}