diff --git a/lib/install.js b/lib/install.js index f9fa2b34bd..cce12ba335 100644 --- a/lib/install.js +++ b/lib/install.js @@ -4,6 +4,8 @@ const fs = require('graceful-fs') const os = require('os') const tar = require('tar') const path = require('path') +const util = require('util') +const stream = require('stream') const crypto = require('crypto') const log = require('npmlog') const semver = require('semver') @@ -12,36 +14,23 @@ const processRelease = require('./process-release') const win = process.platform === 'win32' const getProxyFromURI = require('./proxy') -function install (fs, gyp, argv, callback) { - var release = processRelease(argv, gyp, process.version, process.release) +/** + * @param {import('graceful-fs')} fs + */ - // ensure no double-callbacks happen - function cb (err) { - if (cb.done) { - return - } - cb.done = true - if (err) { - log.warn('install', 'got an error, rolling back install') - // roll-back the install if anything went wrong - gyp.commands.remove([release.versionDir], function () { - callback(err) - }) - } else { - callback(null, release.version) - } - } +async function install (fs, gyp, argv) { + const release = processRelease(argv, gyp, process.version, process.release) // Determine which node dev files version we are installing log.verbose('install', 'input version string %j', release.version) if (!release.semver) { // could not parse the version string with semver - return callback(new Error('Invalid version number: ' + release.version)) + throw new Error('Invalid version number: ' + release.version) } if (semver.lt(release.version, '0.8.0')) { - return callback(new Error('Minimum target version is `0.8.0` or greater. Got: ' + release.version)) + throw new Error('Minimum target version is `0.8.0` or greater. Got: ' + release.version) } // 0.x.y-pre versions are not published yet and cannot be installed. Bail. @@ -49,307 +38,265 @@ function install (fs, gyp, argv, callback) { log.verbose('detected "pre" node version', release.version) if (gyp.opts.nodedir) { log.verbose('--nodedir flag was passed; skipping install', gyp.opts.nodedir) - callback() - } else { - callback(new Error('"pre" versions of node cannot be installed, use the --nodedir flag instead')) + return } - return + throw new Error('"pre" versions of node cannot be installed, use the --nodedir flag instead') } // flatten version into String log.verbose('install', 'installing version: %s', release.versionDir) // the directory where the dev files will be installed - var devDir = path.resolve(gyp.devDir, release.versionDir) + const devDir = path.resolve(gyp.devDir, release.versionDir) // If '--ensure' was passed, then don't *always* install the version; // check if it is already installed, and only install when needed if (gyp.opts.ensure) { log.verbose('install', '--ensure was passed, so won\'t reinstall if already installed') - fs.stat(devDir, function (err) { - if (err) { - if (err.code === 'ENOENT') { - log.verbose('install', 'version not already installed, continuing with install', release.version) - go() - } else if (err.code === 'EACCES') { - eaccesFallback(err) - } else { - cb(err) - } - return + try { + await fs.promises.stat(devDir) + } catch (err) { + if (err.code === 'ENOENT') { + log.verbose('install', 'version not already installed, continuing with install', release.version) + return go().catch(rollback) } - log.verbose('install', 'version is already installed, need to check "installVersion"') - var installVersionFile = path.resolve(devDir, 'installVersion') - fs.readFile(installVersionFile, 'ascii', function (err, ver) { - if (err && err.code !== 'ENOENT') { - return cb(err) - } - var installVersion = parseInt(ver, 10) || 0 - log.verbose('got "installVersion"', installVersion) - log.verbose('needs "installVersion"', gyp.package.installVersion) - if (installVersion < gyp.package.installVersion) { - log.verbose('install', 'version is no good; reinstalling') - go() - } else { - log.verbose('install', 'version is good') - cb() - } - }) - }) - } else { - go() + if (err.code === 'EACCES') { + return eaccesFallback(err) + } + throw err + } + log.verbose('install', 'version is already installed, need to check "installVersion"') + const installVersionFile = path.resolve(devDir, 'installVersion') + let installVersion = 0 + try { + const ver = await fs.promises.readFile(installVersionFile, 'ascii') + installVersion = parseInt(ver, 10) || 0 + } catch (err) { + if (err.code !== 'ENOENT') { + throw err + } + } + log.verbose('got "installVersion"', installVersion) + log.verbose('needs "installVersion"', gyp.package.installVersion) + if (installVersion < gyp.package.installVersion) { + log.verbose('install', 'version is no good; reinstalling') + return go().catch(rollback) + } + log.verbose('install', 'version is good') + return release.version } - function getContentSha (res, callback) { - var shasum = crypto.createHash('sha256') - res.on('data', function (chunk) { - shasum.update(chunk) - }).on('end', function () { - callback(null, shasum.digest('hex')) - }) - } + return go().catch(rollback) - function go () { + async function go () { log.verbose('ensuring nodedir is created', devDir) // first create the dir for the node dev files - fs.mkdir(devDir, { recursive: true }, function (err, created) { - if (err) { - if (err.code === 'EACCES') { - eaccesFallback(err) - } else { - cb(err) - } - return - } + try { + const created = await fs.promises.mkdir(devDir, { recursive: true }) if (created) { log.verbose('created nodedir', created) } - - // now download the node tarball - var tarPath = gyp.opts.tarball - var badDownload = false - var extractCount = 0 - var contentShasums = {} - var expectShasums = {} - - // checks if a file to be extracted from the tarball is valid. - // only .h header files and the gyp files get extracted - function isValid (path) { - var isValid = valid(path) - if (isValid) { - log.verbose('extracted file from tarball', path) - extractCount++ - } else { - // invalid - log.silly('ignoring from tarball', path) - } - return isValid - } - - // download the tarball and extract! - if (tarPath) { - return tar.extract({ - file: tarPath, - strip: 1, - filter: isValid, - cwd: devDir - }).then(afterTarball, cb) + } catch (err) { + if (err.code === 'EACCES') { + return eaccesFallback(err) } + throw err + } - try { - var req = download(gyp, process.env, release.tarballUrl) - } catch (e) { - return cb(e) + // now download the node tarball + const tarPath = gyp.opts.tarball + let extractCount = 0 + const contentShasums = {} + const expectShasums = {} + + // checks if a file to be extracted from the tarball is valid. + // only .h header files and the gyp files get extracted + function isValid (path) { + const isValid = valid(path) + if (isValid) { + log.verbose('extracted file from tarball', path) + extractCount++ + } else { + // invalid + log.silly('ignoring from tarball', path) } + return isValid + } - // something went wrong downloading the tarball? - req.on('error', function (err) { - if (err.code === 'ENOTFOUND') { - return cb(new Error('This is most likely not a problem with node-gyp or the package itself and\n' + - 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' + - 'network settings.')) - } - badDownload = true - cb(err) + // download the tarball and extract! + if (tarPath) { + await tar.extract({ + file: tarPath, + strip: 1, + filter: isValid, + cwd: devDir }) - - req.on('close', function () { - if (extractCount === 0) { - cb(new Error('Connection closed while downloading tarball file')) + } else { + await new Promise((resolve, reject) => { + try { + var req = download(gyp, process.env, release.tarballUrl) + } catch (e) { + return reject(e) } - }) - req.on('response', function (res) { - if (res.statusCode !== 200) { - badDownload = true - cb(new Error(res.statusCode + ' response downloading ' + release.tarballUrl)) - return - } - // content checksum - getContentSha(res, function (_, checksum) { - var filename = path.basename(release.tarballUrl).trim() - contentShasums[filename] = checksum - log.verbose('content checksum', filename, checksum) + // something went wrong downloading the tarball? + req.on('error', function (err) { + if (err.code === 'ENOTFOUND') { + return reject(new Error('This is most likely not a problem with node-gyp or the package itself and\n' + + 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' + + 'network settings.')) + } + reject(err) }) - // start unzipping and untaring - res.pipe(tar.extract({ - strip: 1, - cwd: devDir, - filter: isValid - }).on('close', afterTarball).on('error', cb)) - }) - - // invoked after the tarball has finished being extracted - function afterTarball () { - if (badDownload) { - return - } - if (extractCount === 0) { - return cb(new Error('There was a fatal problem while downloading/extracting the tarball')) - } - log.verbose('tarball', 'done parsing tarball') - var async = 0 - - if (win) { - // need to download node.lib - async++ - downloadNodeLib(deref) - } - - // write the "installVersion" file - async++ - var installVersionPath = path.resolve(devDir, 'installVersion') - fs.writeFile(installVersionPath, gyp.package.installVersion + '\n', deref) + req.on('close', function () { + if (extractCount === 0) { + reject(new Error('Connection closed while downloading tarball file')) + } + }) - // Only download SHASUMS.txt if we downloaded something in need of SHA verification - if (!tarPath || win) { - // download SHASUMS.txt - async++ - downloadShasums(deref) - } + req.on('response', function (res) { + if (res.statusCode !== 200) { + reject(new Error(res.statusCode + ' response downloading ' + release.tarballUrl)) + return + } + // content checksum + const sha256 = new Sha256(function (_, checksum) { + const filename = path.basename(release.tarballUrl).trim() + contentShasums[filename] = checksum + log.verbose('content checksum', filename, checksum) + }) - if (async === 0) { - // no async tasks required - cb() - } + // start unzipping and untaring + res.pipe(sha256.pipe(tar.extract({ + strip: 1, + cwd: devDir, + filter: isValid + }).on('close', resolve).on('error', reject))) + }) + }) + } - function deref (err) { - if (err) { - return cb(err) - } + // invoked after the tarball has finished being extracted - async-- - if (!async) { - log.verbose('download contents checksum', JSON.stringify(contentShasums)) - // check content shasums - for (var k in contentShasums) { - log.verbose('validating download checksum for ' + k, '(%s == %s)', contentShasums[k], expectShasums[k]) - if (contentShasums[k] !== expectShasums[k]) { - cb(new Error(k + ' local checksum ' + contentShasums[k] + ' not match remote ' + expectShasums[k])) - return - } - } - cb() - } - } + if (extractCount === 0) { + throw new Error('There was a fatal problem while downloading/extracting the tarball') + } + log.verbose('tarball', 'done parsing tarball') + + const installVersionPath = path.resolve(devDir, 'installVersion') + await Promise.all([ + // need to download node.lib + ...(win ? downloadNodeLib() : []), + + // write the "installVersion" file + fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'), + + // Only download SHASUMS.txt if we downloaded something in need of SHA verification + ...(!tarPath || win ? [downloadShasums()] : []) + ]) + + log.verbose('download contents checksum', JSON.stringify(contentShasums)) + // check content shasums + for (const k in contentShasums) { + log.verbose('validating download checksum for ' + k, '(%s == %s)', contentShasums[k], expectShasums[k]) + if (contentShasums[k] !== expectShasums[k]) { + throw new Error(k + ' local checksum ' + contentShasums[k] + ' not match remote ' + expectShasums[k]) } + } + return release.version - function downloadShasums (done) { - log.verbose('check download content checksum, need to download `SHASUMS256.txt`...') - log.verbose('checksum url', release.shasumsUrl) + function downloadShasums () { + log.verbose('check download content checksum, need to download `SHASUMS256.txt`...') + log.verbose('checksum url', release.shasumsUrl) + return new Promise((resolve, reject) => { try { var req = download(gyp, process.env, release.shasumsUrl) } catch (e) { - return cb(e) + return reject(e) } - req.on('error', done) + req.on('error', reject) req.on('response', function (res) { if (res.statusCode !== 200) { - done(new Error(res.statusCode + ' status code downloading checksum')) + reject(new Error(res.statusCode + ' status code downloading checksum')) return } - var chunks = [] + const chunks = [] res.on('data', function (chunk) { chunks.push(chunk) }) res.on('end', function () { - var lines = Buffer.concat(chunks).toString().trim().split('\n') + const lines = Buffer.concat(chunks).toString().trim().split('\n') lines.forEach(function (line) { - var items = line.trim().split(/\s+/) + const items = line.trim().split(/\s+/) if (items.length !== 2) { return } // 0035d18e2dcf9aad669b1c7c07319e17abfe3762 ./node-v0.11.4.tar.gz - var name = items[1].replace(/^\.\//, '') + const name = items[1].replace(/^\.\//, '') expectShasums[name] = items[0] }) log.verbose('checksum data', JSON.stringify(expectShasums)) - done() + resolve() }) }) - } - - function downloadNodeLib (done) { - log.verbose('on Windows; need to download `' + release.name + '.lib`...') - var archs = ['ia32', 'x64', 'arm64'] - var async = archs.length - archs.forEach(function (arch) { - var dir = path.resolve(devDir, arch) - var targetLibPath = path.resolve(dir, release.name + '.lib') - var libUrl = release[arch].libUrl - var libPath = release[arch].libPath - var name = arch + ' ' + release.name + '.lib' - log.verbose(name, 'dir', dir) - log.verbose(name, 'url', libUrl) - - fs.mkdir(dir, { recursive: true }, function (err) { - if (err) { - return done(err) - } - log.verbose('streaming', name, 'to:', targetLibPath) + }) + } - try { - var req = download(gyp, process.env, libUrl, cb) - } catch (e) { - return cb(e) - } + function downloadNodeLib () { + log.verbose('on Windows; need to download `' + release.name + '.lib`...') + const archs = ['ia32', 'x64', 'arm64'] + return archs.map(async function (arch) { + const dir = path.resolve(devDir, arch) + const targetLibPath = path.resolve(dir, release.name + '.lib') + const { libUrl, libPath } = release[arch] + const name = arch + ' ' + release.name + '.lib' + log.verbose(name, 'dir', dir) + log.verbose(name, 'url', libUrl) + + await fs.promises.mkdir(dir, { recursive: true }) + log.verbose('streaming', name, 'to:', targetLibPath) + + return new Promise((resolve, reject) => { + try { + var req = download(gyp, process.env, libUrl) + } catch (e) { + return reject(e) + } - req.on('error', done) - req.on('response', function (res) { - if (res.statusCode === 403 || res.statusCode === 404) { - if (arch === 'arm64') { - // Arm64 is a newer platform on Windows and not all node distributions provide it. - log.verbose(`${name} was not found in ${libUrl}`) - } else { - log.warn(`${name} was not found in ${libUrl}`) - } - return - } else if (res.statusCode !== 200) { - done(new Error(res.statusCode + ' status code downloading ' + name)) - return + req.on('error', reject) + req.on('response', function (res) { + if (res.statusCode === 403 || res.statusCode === 404) { + if (arch === 'arm64') { + // Arm64 is a newer platform on Windows and not all node distributions provide it. + log.verbose(`${name} was not found in ${libUrl}`) + } else { + log.warn(`${name} was not found in ${libUrl}`) } + return resolve() + } else if (res.statusCode !== 200) { + reject(new Error(res.statusCode + ' status code downloading ' + name)) + return + } - getContentSha(res, function (_, checksum) { - contentShasums[libPath] = checksum - log.verbose('content checksum', libPath, checksum) - }) - - var ws = fs.createWriteStream(targetLibPath) - ws.on('error', cb) - req.pipe(ws) + const sha256 = new Sha256(function (_, checksum) { + contentShasums[libPath] = checksum + log.verbose('content checksum', libPath, checksum) }) - req.on('end', function () { --async || done() }) + + const ws = fs.createWriteStream(targetLibPath) + ws.on('close', resolve).on('error', reject) + res.pipe(sha256.pipe(ws)) }) }) - } // downloadNodeLib() - }) // mkdir() + }) + } // downloadNodeLib() } // go() /** @@ -358,10 +305,17 @@ function install (fs, gyp, argv, callback) { function valid (file) { // header files - var extname = path.extname(file) + const extname = path.extname(file) return extname === '.h' || extname === '.gypi' } + async function rollback (err) { + log.warn('install', 'got an error, rolling back install') + // roll-back the install if anything went wrong + await util.promisify(gyp.commands.remove)([release.versionDir]) + throw err + } + /** * The EACCES fallback is a workaround for npm's `sudo` behavior, where * it drops the permissions before invoking any child processes (like @@ -371,14 +325,14 @@ function install (fs, gyp, argv, callback) { * the compilation will succeed... */ - function eaccesFallback (err) { - var noretry = '--node_gyp_internal_noretry' + async function eaccesFallback (err) { + const noretry = '--node_gyp_internal_noretry' if (argv.indexOf(noretry) !== -1) { - return cb(err) + throw err } - var tmpdir = os.tmpdir() + const tmpdir = os.tmpdir() gyp.devDir = path.resolve(tmpdir, '.node-gyp') - var userString = '' + let userString = '' try { // os.userInfo can fail on some systems, it's not critical here userString = ` ("${os.userInfo().username}")` @@ -389,14 +343,32 @@ function install (fs, gyp, argv, callback) { log.verbose('tmpdir == cwd', 'automatically will remove dev files after to save disk space') gyp.todo.push({ name: 'remove', args: argv }) } - gyp.commands.install([noretry].concat(argv), cb) + return util.promisify(gyp.commands.install)([noretry].concat(argv)) + } +} + +class Sha256 extends stream.Transform { + constructor (callback) { + super() + this._callback = callback + this._digester = crypto.createHash('sha256') + } + + _transform (chunk, _, callback) { + this._digester.update(chunk) + callback(chunk) + } + + _flush (callback) { + this._callback(null, this._digester.digest('hex')) + callback() } } function download (gyp, env, url) { log.http('GET', url) - var requestOpts = { + const requestOpts = { uri: url, headers: { 'User-Agent': 'node-gyp v' + gyp.version + ' (node ' + process.version + ')', @@ -404,13 +376,13 @@ function download (gyp, env, url) { } } - var cafile = gyp.opts.cafile + const cafile = gyp.opts.cafile if (cafile) { requestOpts.ca = readCAFile(cafile) } // basic support for a proxy server - var proxyUrl = getProxyFromURI(gyp, env, url) + const proxyUrl = getProxyFromURI(gyp, env, url) if (proxyUrl) { if (/^https?:\/\//i.test(proxyUrl)) { log.verbose('download', 'using proxy url: "%s"', proxyUrl) @@ -420,7 +392,7 @@ function download (gyp, env, url) { } } - var req = request(requestOpts) + const req = request(requestOpts) req.on('response', function (res) { log.http(res.statusCode, url) }) @@ -431,17 +403,17 @@ function download (gyp, env, url) { function readCAFile (filename) { // The CA file can contain multiple certificates so split on certificate // boundaries. [\S\s]*? is used to match everything including newlines. - var ca = fs.readFileSync(filename, 'utf8') - var re = /(-----BEGIN CERTIFICATE-----[\S\s]*?-----END CERTIFICATE-----)/g + const ca = fs.readFileSync(filename, 'utf8') + const re = /(-----BEGIN CERTIFICATE-----[\S\s]*?-----END CERTIFICATE-----)/g return ca.match(re) } module.exports = function (gyp, argv, callback) { - return install(fs, gyp, argv, callback) + return install(fs, gyp, argv).then(callback.bind(undefined, null), callback) } module.exports.test = { - download: download, - install: install, - readCAFile: readCAFile + download, + install, + readCAFile } module.exports.usage = 'Install node development files for the specified node version.' diff --git a/test/test-download.js b/test/test-download.js index fe373e3280..ea38d03bb9 100644 --- a/test/test-download.js +++ b/test/test-download.js @@ -1,7 +1,8 @@ 'use strict' -const test = require('tap').test +const { test } = require('tap') const fs = require('fs') +const util = require('util') const path = require('path') const http = require('http') const https = require('https') @@ -206,7 +207,7 @@ test('check certificate splitting', function (t) { // only run this test if we are running a version of Node with predictable version path behavior -test('download headers (actual)', function (t) { +test('download headers (actual)', async (t) => { if (process.env.FAST_TEST || process.release.name !== 'node' || semver.prerelease(process.version) !== null || @@ -214,55 +215,42 @@ test('download headers (actual)', function (t) { return t.skip('Skipping actual download of headers due to test environment configuration') } - t.plan(17) + t.plan(12) const expectedDir = path.join(devDir, process.version.replace(/^v/, '')) - rimraf(expectedDir, (err) => { - t.ifError(err) - - const prog = gyp() - prog.parseArgv([]) - prog.devDir = devDir - log.level = 'warn' - install(prog, [], (err) => { - t.ifError(err) - - fs.readFile(path.join(expectedDir, 'installVersion'), 'utf8', (err, data) => { - t.ifError(err) - t.strictEqual(data, '9\n', 'correct installVersion') - }) - - fs.readdir(path.join(expectedDir, 'include/node'), (err, list) => { - t.ifError(err) - - t.ok(list.includes('common.gypi')) - t.ok(list.includes('config.gypi')) - t.ok(list.includes('node.h')) - t.ok(list.includes('node_version.h')) - t.ok(list.includes('openssl')) - t.ok(list.includes('uv')) - t.ok(list.includes('uv.h')) - t.ok(list.includes('v8-platform.h')) - t.ok(list.includes('v8.h')) - t.ok(list.includes('zlib.h')) - }) - - fs.readFile(path.join(expectedDir, 'include/node/node_version.h'), 'utf8', (err, contents) => { - t.ifError(err) - - const lines = contents.split('\n') - - // extract the 3 version parts from the defines to build a valid version string and - // and check them against our current env version - const version = ['major', 'minor', 'patch'].reduce((version, type) => { - const re = new RegExp(`^#define\\sNODE_${type.toUpperCase()}_VERSION`) - const line = lines.find((l) => re.test(l)) - const i = line ? parseInt(line.replace(/^[^0-9]+([0-9]+).*$/, '$1'), 10) : 'ERROR' - return `${version}${type !== 'major' ? '.' : 'v'}${i}` - }, '') - - t.strictEqual(version, process.version) - }) - }) - }) + await util.promisify(rimraf)(expectedDir) + + const prog = gyp() + prog.parseArgv([]) + prog.devDir = devDir + log.level = 'warn' + await util.promisify(install)(prog, []) + + const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8') + t.strictEqual(data, '9\n', 'correct installVersion') + + const list = await fs.promises.readdir(path.join(expectedDir, 'include/node')) + t.ok(list.includes('common.gypi')) + t.ok(list.includes('config.gypi')) + t.ok(list.includes('node.h')) + t.ok(list.includes('node_version.h')) + t.ok(list.includes('openssl')) + t.ok(list.includes('uv')) + t.ok(list.includes('uv.h')) + t.ok(list.includes('v8-platform.h')) + t.ok(list.includes('v8.h')) + t.ok(list.includes('zlib.h')) + + const lines = (await fs.promises.readFile(path.join(expectedDir, 'include/node/node_version.h'), 'utf8')).split('\n') + + // extract the 3 version parts from the defines to build a valid version string and + // and check them against our current env version + const version = ['major', 'minor', 'patch'].reduce((version, type) => { + const re = new RegExp(`^#define\\sNODE_${type.toUpperCase()}_VERSION`) + const line = lines.find((l) => re.test(l)) + const i = line ? parseInt(line.replace(/^[^0-9]+([0-9]+).*$/, '$1'), 10) : 'ERROR' + return `${version}${type !== 'major' ? '.' : 'v'}${i}` + }, '') + + t.strictEqual(version, process.version) }) diff --git a/test/test-install.js b/test/test-install.js index c3317155e0..5039dc992e 100644 --- a/test/test-install.js +++ b/test/test-install.js @@ -1,38 +1,46 @@ 'use strict' -const test = require('tap').test -const install = require('../lib/install').test.install +const { test } = require('tap') +const { test: { install } } = require('../lib/install') +const log = require('npmlog') -require('npmlog').level = 'error' // we expect a warning +log.level = 'error' // we expect a warning -test('EACCES retry once', function (t) { +test('EACCES retry once', async (t) => { t.plan(3) - var fs = {} - fs.stat = function (path, cb) { - var err = new Error() - err.code = 'EACCES' - cb(err) - t.ok(true) + const fs = { + promises: { + stat (_) { + const err = new Error() + err.code = 'EACCES' + t.ok(true) + throw err + } + } } - var gyp = {} - gyp.devDir = __dirname - gyp.opts = {} - gyp.opts.ensure = true - gyp.commands = {} - gyp.commands.install = function (argv, cb) { - install(fs, gyp, argv, cb) - } - gyp.commands.remove = function (argv, cb) { - cb() + const Gyp = { + devDir: __dirname, + opts: { + ensure: true + }, + commands: { + install (argv, cb) { + install(fs, Gyp, argv).then(cb, cb) + }, + remove (_, cb) { + cb() + } + } } - gyp.commands.install([], function (err) { + try { + await install(fs, Gyp, []) + } catch (err) { t.ok(true) if (/"pre" versions of node cannot be installed/.test(err.message)) { t.ok(true) - t.ok(true) } - }) + } })