diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ad5768..5b164790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## master +## 2.0.2 +- Support private ACL for S3 buckets [#923](https://github.com/mapbox/node-pre-gyp/pull/923) + ## 2.0.1 - Update abi_crosswalk.json for abi 137 / node 24 (https://github.com/mapbox/node-pre-gyp/pull/904) diff --git a/README.md b/README.md index 5aef79a6..3dedd9cc 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Options include: - `--target=0.4.0`: Pass the target node or node-webkit version to compile against - `--target_arch=ia32`: Pass the target arch and override the host `arch`. Any value that is [supported by Node.js](https://nodejs.org/api/os.html#osarch) is valid. - `--target_platform=win32`: Pass the target platform and override the host `platform`. Valid values are `linux`, `darwin`, `win32`, `sunos`, `freebsd`, `openbsd`, and `aix`. + - `--acl=`: Set the S3 ACL when publishing binaries (e.g., `public-read`, `private`). Overrides the `binary.acl` setting in package.json. Both `--build-from-source` and `--fallback-to-build` can be passed alone or they can provide values. You can pass `--fallback-to-build=false` to override the option as declared in package.json. In addition to being able to pass `--build-from-source` you can also pass `--build-from-source=myapp` where `myapp` is the name of your module. @@ -185,6 +186,33 @@ Your S3 server region. Set `s3ForcePathStyle` to true if the endpoint url should not be prefixed with the bucket name. If false (default), the server endpoint would be constructed as `bucket_name.your_server.com`. +###### acl + +The S3 Access Control List (ACL) to apply when publishing binaries. Defaults to `'public-read'` for backward compatibility. Common values include: + +- `public-read` - (default) Binary is publicly accessible by anyone +- `private` - Binary requires AWS credentials to download +- `authenticated-read` - Any authenticated AWS user can download +- `bucket-owner-read` - Bucket owner gets READ access +- `bucket-owner-full-control` - Bucket owner gets FULL_CONTROL access + +**For private binaries:** +- Users installing your package will need AWS credentials configured (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables) +- The `aws-sdk` package must be available at install time +- If authentication fails, node-pre-gyp will fall back to building from source (if `--fallback-to-build` is specified) + +You can also specify the ACL via command-line flag: `node-pre-gyp publish --acl=private` + +Example for private binaries: +```json +"binary": { + "module_name": "your_module", + "module_path": "./lib/binding/", + "host": "https://your-bucket.s3.us-east-1.amazonaws.com", + "acl": "private" +} +``` + ##### The `binary` object has optional properties ###### remote_path diff --git a/lib/install.js b/lib/install.js index 119af6db..51839cc8 100644 --- a/lib/install.js +++ b/lib/install.js @@ -10,6 +10,8 @@ const log = require('./util/log.js'); const existsAsync = fs.exists || path.exists; const versioning = require('./util/versioning.js'); const napi = require('./util/napi.js'); +const s3_setup = require('./util/s3_setup.js'); +const url = require('url'); // for fetching binaries const fetch = require('node-fetch'); const tar = require('tar'); @@ -23,6 +25,65 @@ try { // do nothing } +function place_binary_authenticated(opts, targetDir, callback) { + log.info('install', 'Attempting authenticated S3 download'); + + // Check if AWS credentials are available + if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { + const err = new Error('Binary is private but AWS credentials not found. Please configure AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables, or use --fallback-to-build to compile from source.'); + err.statusCode = 403; + return callback(err); + } + + try { + const config = s3_setup.detect(opts); + const s3 = s3_setup.get_s3(config); + const key_name = url.resolve(config.prefix, opts.package_name); + + log.info('install', 'Downloading from S3:', config.bucket, key_name); + + const s3_opts = { + Bucket: config.bucket, + Key: key_name + }; + + s3.getObject(s3_opts, (err, data) => { + if (err) { + log.error('install', 'Authenticated S3 download failed:', err.message); + return callback(err); + } + + log.info('install', 'Authenticated download successful, extracting...'); + + const { Readable } = require('stream'); + const dataStream = Readable.from(data.Body); + + let extractions = 0; + const countExtractions = (entry) => { + extractions += 1; + log.info('install', `unpacking ${entry.path}`); + }; + + dataStream.pipe(extract(targetDir, countExtractions)) + .on('error', (e) => { + callback(e); + }) + .on('close', () => { + log.info('install', `extracted file count: ${extractions}`); + callback(); + }); + }); + } catch (e) { + if (e.code === 'MODULE_NOT_FOUND' && e.message.includes('aws-sdk')) { + const err = new Error('Binary is private and requires aws-sdk for authenticated download. Please run: npm install aws-sdk'); + err.statusCode = 403; + return callback(err); + } + log.error('install', 'Error setting up authenticated download:', e.message); + callback(e); + } +} + function place_binary(uri, targetDir, opts, callback) { log.log('GET', uri); @@ -63,6 +124,14 @@ function place_binary(uri, targetDir, opts, callback) { fetch(sanitized, { agent }) .then((res) => { if (!res.ok) { + // If we get 403 Forbidden, the binary might be private - try authenticated download + if (res.status === 403) { + log.info('install', 'Received 403 Forbidden - attempting authenticated download'); + // Call place_binary_authenticated and return a special marker + // to prevent the promise chain from calling callback again + place_binary_authenticated(opts, targetDir, callback); + return { authenticated: true }; + } throw new Error(`response status ${res.status} ${res.statusText} on ${sanitized}`); } const dataStream = res.body; @@ -87,6 +156,9 @@ function place_binary(uri, targetDir, opts, callback) { }); }) .then((text) => { + if (text && text.authenticated) { + return; // Don't call callback - place_binary_authenticated will handle it + } log.info(text); callback(); }) diff --git a/lib/mock/s3.js b/lib/mock/s3.js index e485e3bd..9633e5d3 100644 --- a/lib/mock/s3.js +++ b/lib/mock/s3.js @@ -36,6 +36,9 @@ function s3_mock() { }, putObject(params, callback) { return s3.putObject(params, wcb(callback)); + }, + getObject(params, callback) { + return s3.getObject(params, wcb(callback)); } }; } diff --git a/lib/node-pre-gyp.js b/lib/node-pre-gyp.js index 26c0e512..bc80dd90 100644 --- a/lib/node-pre-gyp.js +++ b/lib/node-pre-gyp.js @@ -98,7 +98,8 @@ proto.configDefs = { debug: Boolean, // 'build' directory: String, // bin proxy: String, // 'install' - loglevel: String // everywhere + loglevel: String, // everywhere + acl: String // 'publish' - S3 ACL for published binaries }; /** diff --git a/lib/publish.js b/lib/publish.js index 3b2e08cd..97067ce4 100644 --- a/lib/publish.js +++ b/lib/publish.js @@ -43,12 +43,13 @@ function publish(gyp, argv, callback) { // the object does not already exist log.info('publish', 'Preparing to put object'); const s3_put_opts = { - ACL: 'public-read', + ACL: opts.acl, Body: fs.createReadStream(tarball), Key: key_name, Bucket: config.bucket }; - log.info('publish', 'Putting object', s3_put_opts.ACL, s3_put_opts.Bucket, s3_put_opts.Key); + log.info('publish', 'Putting object with ACL:', s3_put_opts.ACL); + log.info('publish', 'Bucket:', s3_put_opts.Bucket, 'Key:', s3_put_opts.Key); try { s3.putObject(s3_put_opts, (err2, resp) => { log.info('publish', 'returned from putting object'); diff --git a/lib/util/s3_setup.js b/lib/util/s3_setup.js index 0095cd34..6fd3c924 100644 --- a/lib/util/s3_setup.js +++ b/lib/util/s3_setup.js @@ -84,6 +84,9 @@ module.exports.get_s3 = function(config) { }, putObject(params, callback) { return s3.putObject(params, callback); + }, + getObject(params, callback) { + return s3.getObject(params, callback); } }; }; diff --git a/lib/util/versioning.js b/lib/util/versioning.js index b65e84e7..b69b367e 100644 --- a/lib/util/versioning.js +++ b/lib/util/versioning.js @@ -307,7 +307,8 @@ module.exports.evaluate = function(package_json, options, napi_build_version) { toolset: options.toolset || '', // address https://github.com/mapbox/node-pre-gyp/issues/119 bucket: package_json.binary.bucket, region: package_json.binary.region, - s3ForcePathStyle: package_json.binary.s3ForcePathStyle || false + s3ForcePathStyle: package_json.binary.s3ForcePathStyle || false, + acl: options.acl || package_json.binary.acl || 'public-read' }; // support host mirror with npm config `--{module_name}_binary_host_mirror` // e.g.: https://github.com/node-inspector/v8-profiler/blob/master/package.json#L25 diff --git a/package-lock.json b/package-lock.json index 4f29063b..f18ab83f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mapbox/node-pre-gyp", - "version": "2.0.1", + "version": "2.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mapbox/node-pre-gyp", - "version": "2.0.1", + "version": "2.0.2", "license": "BSD-3-Clause", "dependencies": { "consola": "^3.2.3", @@ -1118,9 +1118,9 @@ } }, "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, "license": "MIT", "dependencies": { @@ -3772,9 +3772,9 @@ } }, "node_modules/eslint/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, "license": "MIT", "dependencies": { @@ -5322,9 +5322,9 @@ "dev": true }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", "dependencies": { @@ -6435,9 +6435,9 @@ } }, "node_modules/rimraf/node_modules/glob": { - "version": "10.4.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.2.tgz", - "integrity": "sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", @@ -6450,9 +6450,6 @@ "bin": { "glob": "dist/esm/bin.mjs" }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, "funding": { "url": "https://github.com/sponsors/isaacs" } diff --git a/package.json b/package.json index 1b744cbd..81cfa20e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@mapbox/node-pre-gyp", "description": "Node.js native addon binary install tool", - "version": "2.0.1", + "version": "2.0.2", "keywords": [ "native", "addon", @@ -61,5 +61,8 @@ "test": "tape test/*test.js", "test:s3": "tape test/s3.test.js", "bucket": "node scripts/set-bucket.js" + }, + "overrides": { + "js-yaml": "^3.14.2" } } diff --git a/test/private-binary.test.js b/test/private-binary.test.js new file mode 100644 index 00000000..9491a265 --- /dev/null +++ b/test/private-binary.test.js @@ -0,0 +1,168 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const test = require('tape'); +const nock = require('nock'); +const install = require('../lib/install.js'); +const os = require('os'); +const rimraf = require('rimraf'); + +// Dummy tar.gz data - contains a blank directory +const targz = 'H4sICPr8u1oCA3gudGFyANPTZ6A5MDAwMDc1VQDTZhAaCGA0hGNobGRqZm5uZmxupGBgaGhiZsKgYMpAB1BaXJJYBHRKYk5pcioedeUZqak5+D2J5CkFhlEwCkbBKBjkAAAyG1ofAAYAAA=='; + +// Determine the project root (where package.json with @mapbox/node-pre-gyp exists) +const projectRoot = path.join(__dirname, '..'); + +// Helper to clean mock S3 directory +function cleanMockS3() { + const mockDir = path.join(os.tmpdir(), 'mock'); + if (fs.existsSync(mockDir)) { + rimraf.sync(mockDir); + } +} + +test('should fallback to authenticated download on 403 Forbidden', (t) => { + // Clean mock S3 to ensure deterministic behavior + cleanMockS3(); + process.env.node_pre_gyp_mock_s3 = 'true'; + process.env.AWS_ACCESS_KEY_ID = 'mock-key'; + process.env.AWS_SECRET_ACCESS_KEY = 'mock-secret'; + + const origin = 'https://npg-mock-bucket.s3.us-east-1.amazonaws.com'; + const run = require('./run.util.js'); + const app = { name: 'app1', args: '' }; + + // First build and publish to mock S3 to create the file + run('node-pre-gyp', 'configure build package publish', '', app, {}, (buildErr) => { + t.ifError(buildErr, 'Build and publish to mock S3 should succeed'); + + nock.cleanAll(); + const publicScope = nock(origin) + .get(/\/node-pre-gyp\/node-pre-gyp-test-app1\/v0.1.0\/Release\/node-v\d+-\S+.tar.gz/) + .reply(403, 'Forbidden'); + + const appDir = path.join(projectRoot, 'test', 'app1'); + process.chdir(appDir); + const installOpts = { + opts: { + 'build-from-source': false, + 'update-binary': true + }, + package_json: JSON.parse(fs.readFileSync('./package.json')) + }; + installOpts.package_json.binary.host = origin; + + // Run install - should get 403 from HTTP, then succeed with authenticated S3 + install(installOpts, [], (err) => { + delete process.env.node_pre_gyp_mock_s3; + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + + t.ok(publicScope.isDone(), 'Public HTTPS request was attempted (got 403)'); + t.ifError(err, 'Authenticated S3 download should succeed after 403'); + + nock.cleanAll(); + cleanMockS3(); + t.end(); + }); + }); +}); + +test('should fail gracefully when 403 and no AWS credentials', (t) => { + // Ensure no AWS credentials + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + delete process.env.node_pre_gyp_mock_s3; + + const origin = 'https://npg-mock-bucket.s3.us-east-1.amazonaws.com'; + + nock.cleanAll(); + const publicScope = nock(origin) + .get(/\/node-pre-gyp\/node-pre-gyp-test-app1\/v0.1.0\/Release\/node-v\d+-\S+.tar.gz/) + .reply(403, 'Forbidden'); + + const opts = { + opts: { + 'build-from-source': false, + 'update-binary': true + } + }; + + // cd into app directory from project root + const appDir = path.join(projectRoot, 'test', 'app1'); + process.chdir(appDir); + opts.package_json = JSON.parse(fs.readFileSync('./package.json')); + opts.package_json.binary.host = origin; + + install(opts, [], (err) => { + t.ok(err, 'Should error without credentials'); + t.ok(err.message.includes('AWS credentials not found'), 'Error should mention missing credentials'); + t.equal(err.statusCode, 403, 'Error should have 403 status code'); + t.ok(publicScope.isDone(), 'Public HTTPS request was attempted'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('should succeed with public binary (no 403)', (t) => { + const origin = 'https://npg-mock-bucket.s3.us-east-1.amazonaws.com'; + nock.cleanAll(); + const scope = nock(origin) + .get(/\/node-pre-gyp\/node-pre-gyp-test-app1\/v0.1.0\/Release\/node-v\d+-\S+.tar.gz/) + .reply(200, Buffer.from(targz, 'base64')); + + const opts = { + opts: { + 'build-from-source': false, + 'update-binary': true + } + }; + + // cd into app directory from project root + const appDir = path.join(projectRoot, 'test', 'app1'); + process.chdir(appDir); + opts.package_json = JSON.parse(fs.readFileSync('./package.json')); + opts.package_json.binary.host = origin; + + install(opts, [], (err) => { + t.ifError(err, 'Public binary install should succeed'); + t.ok(scope.isDone(), 'Public download was completed'); + + nock.cleanAll(); + t.end(); + }); +}); + +test('should handle 404 without triggering authenticated fallback', (t) => { + const origin = 'https://npg-mock-bucket.s3.us-east-1.amazonaws.com'; + + nock.cleanAll(); + const scope = nock(origin) + .get(/\/node-pre-gyp\/node-pre-gyp-test-app1\/v0.1.0\/Release\/node-v\d+-\S+.tar.gz/) + .reply(404, 'Not Found'); + + const opts = { + opts: { + 'build-from-source': false, + 'update-binary': true + } + }; + + // cd into app directory from project root + const appDir = path.join(projectRoot, 'test', 'app1'); + process.chdir(appDir); + opts.package_json = JSON.parse(fs.readFileSync('./package.json')); + opts.package_json.binary.host = origin; + + install(opts, [], (err) => { + t.ok(err, 'Should error with 404'); + t.ok(err.message.includes('404'), 'Error message should mention 404'); + t.notOk(err.message.includes('credentials'), 'Should not mention credentials for 404'); + t.ok(scope.isDone(), 'HTTP request was completed'); + + nock.cleanAll(); + t.end(); + }); +}); diff --git a/test/versioning.test.js b/test/versioning.test.js index 45541999..77e3b75f 100644 --- a/test/versioning.test.js +++ b/test/versioning.test.js @@ -100,6 +100,56 @@ test('should throw when custom node target is not found in abi_crosswalk file', } }); +test('should default ACL to public-read when not specified', (t) => { + const mock_package_json = { + 'name': 'test', + 'main': 'test.js', + 'version': '0.1.0', + 'binary': { + 'module_name': 'test', + 'module_path': './lib/binding/', + 'host': 'https://some-bucket.s3.us-east-1.amazonaws.com' + } + }; + const opts = versioning.evaluate(mock_package_json, {}); + t.equal(opts.acl, 'public-read', 'ACL should default to public-read'); + t.end(); +}); + +test('should use ACL from package.json binary.acl', (t) => { + const mock_package_json = { + 'name': 'test', + 'main': 'test.js', + 'version': '0.1.0', + 'binary': { + 'module_name': 'test', + 'module_path': './lib/binding/', + 'host': 'https://some-bucket.s3.us-east-1.amazonaws.com', + 'acl': 'private' + } + }; + const opts = versioning.evaluate(mock_package_json, {}); + t.equal(opts.acl, 'private', 'ACL should be read from package.json binary.acl'); + t.end(); +}); + +test('should allow CLI flag to override package.json ACL', (t) => { + const mock_package_json = { + 'name': 'test', + 'main': 'test.js', + 'version': '0.1.0', + 'binary': { + 'module_name': 'test', + 'module_path': './lib/binding/', + 'host': 'https://some-bucket.s3.us-east-1.amazonaws.com', + 'acl': 'private' + } + }; + const opts = versioning.evaluate(mock_package_json, { acl: 'authenticated-read' }); + t.equal(opts.acl, 'authenticated-read', 'CLI flag should override package.json ACL'); + t.end(); +}); + test('should throw when custom node target is not semver', (t) => { try { versioning.get_runtime_abi('node', '1.2.3.4');