chore: create smoke-publish-test.sh scripts

lukekarrys · lukekarrys · commit d718afa56b76 · 2024-05-08T14:15:32.000-07:00
This makes it easier to debug this test locally. Previously this was
avoided because the test will install npm globally and prune deps which
can be disruptive to a local checkout. This is somewhat mitigated now
with some cleanup and better messaging when run locally.
diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml
@@ -217,27 +217,8 @@ jobs:
         run: node scripts/git-dirty.js
       - name: Reset Deps
         run: node scripts/resetdeps.js
-      - name: Pack
-        env:
-          SMOKE_PUBLISH_NPM: 1
-        run: |
-          NPM_VERSION="$(node . --version)-$GITHUB_SHA.0"
-          node . version $NPM_VERSION --ignore-scripts
-          node scripts/publish.js --pack-destination=$RUNNER_TEMP
-          export SMOKE_PUBLISH_TARBALL="$RUNNER_TEMP/npm-$NPM_VERSION.tgz"
-          node . install --global $SMOKE_PUBLISH_TARBALL
-          node . install -w smoke-tests --ignore-scripts --no-audit --no-fund
-          # call installed npm instead of local source since we are testing
-          # the packed tarball that we just installed globally
-          NPM_GLOBAL_VERSION="$(npm --version)"
-          npm help
-          if [ "$NPM_GLOBAL_VERSION" == "$NPM_VERSION" ]; then
-            npm test -w smoke-tests --ignore-scripts
-          else
-            echo "global npm is not the correct version for smoke-publish"
-            echo "found: $NPM_GLOBAL_VERSION, expected: $NPM_VERSION"
-            exit 1
-          fi
+      - name: Smoke Publish
+        run: ./scripts/smoke-publish-test.sh
       - name: Conclude Check
         uses: LouisBrunner/checks-action@v1.6.0
         if: always()
diff --git a/docs/bin/build.js b/docs/bin/build.js
@@ -1,17 +1,3 @@
-if (
-  process.env.SMOKE_PUBLISH_NPM &&
-  !require('semver').satisfies(process.version, require('../package.json').engines.node)
-) {
-  // The docs tooling is kept in sync between releases and dependencies that are not compatible
-  // with the lower bound of npm@8 engines are used. When we run the SMOKE_PUBLISH_NPM we are
-  // testing that npm is able to pack and install itself locally and then run its own smoke tests.
-  // Packing will run this script automatically so in the cases where the node version is
-  // not compatible, it is ok to bail on this script since the generated docs are not used in
-  // the smoke tests.
-  console.log(`Skipping docs build due to SMOKE_PUBLISH_NPM and ${process.version}`)
-  return
-}
-
 const run = require('../lib/build.js')
 const { paths } = require('../lib/index')
 
diff --git a/mock-registry/lib/index.js b/mock-registry/lib/index.js
@@ -4,6 +4,17 @@ const npa = require('npm-package-arg')
 const Nock = require('nock')
 const stringify = require('json-stringify-safe')
 
+const logReq = (req, ...keys) => {
+  const obj = JSON.parse(stringify(req))
+  const res = {}
+  for (const [k, v] of Object.entries(obj)) {
+    if (!keys.includes(k)) {
+      res[k] = v
+    }
+  }
+  return stringify(res, null, 2)
+}
+
 class MockRegistry {
   #tap
   #nock
@@ -40,7 +51,8 @@ class MockRegistry {
         // mocked with a 404, 500, etc.
         // XXX: this is opt-in currently because it breaks some existing CLI
         // tests. We should work towards making this the default for all tests.
-        t.fail(`Unmatched request: ${stringify(req, null, 2)}`)
+        t.comment(logReq(req, 'interceptors', 'socket', 'response', '_events'))
+        t.fail(`Unmatched request: ${req.method} ${req.path}`)
       }
     }
 
@@ -357,7 +369,7 @@ class MockRegistry {
     })
   }
 
-  async package ({ manifest, times = 1, query, tarballs }) {
+  async package ({ manifest, times = 1, query, tarballs, tarballTimes = 1 }) {
     let nock = this.nock
     const spec = npa(manifest.name)
     nock = nock.get(this.fullPath(`/${spec.escapedName}`)).times(times)
@@ -368,17 +380,17 @@ class MockRegistry {
     if (tarballs) {
       for (const [version, tarball] of Object.entries(tarballs)) {
         const m = manifest.versions[version]
-        nock = await this.tarball({ manifest: m, tarball })
+        nock = await this.tarball({ manifest: m, tarball, times: tarballTimes })
       }
     }
     this.nock = nock
   }
 
-  async tarball ({ manifest, tarball }) {
+  async tarball ({ manifest, tarball, times = 1 }) {
     const nock = this.nock
     const dist = new URL(manifest.dist.tarball)
     const tar = await pacote.tarball(tarball, { Arborist })
-    nock.get(this.fullPath(dist.pathname)).reply(200, tar)
+    nock.get(this.fullPath(dist.pathname)).times(times).reply(200, tar)
     return nock
   }
 
diff --git a/scripts/publish.js b/scripts/publish.js
@@ -72,8 +72,9 @@ const getPublishes = async ({ force }) => {
 }
 
 const main = async (opts) => {
-  const packOnly = opts.pack || opts.packDestination
-  const publishes = await getPublishes({ force: packOnly })
+  const { isLocal, smokePublish, packDestination } = opts
+  const isPack = !!packDestination
+  const publishes = await getPublishes({ force: isPack })
 
   if (!publishes.length) {
     throw new Error(
@@ -88,12 +89,12 @@ const main = async (opts) => {
   }
 
   const confirmMessage = [
-    `Ready to ${packOnly ? 'pack' : 'publish'} the following packages:`,
+    `Ready to ${isPack ? 'pack' : 'publish'} the following packages:`,
     table.toString(),
-    packOnly ? null : 'Ok to proceed? ',
+    isPack ? null : 'Ok to proceed? ',
   ].filter(Boolean).join('\n')
 
-  if (packOnly) {
+  if (isPack) {
     log.info(confirmMessage)
   } else {
     const confirm = await read({ prompt: confirmMessage, default: 'y' })
@@ -116,21 +117,26 @@ const main = async (opts) => {
 
   await npm('prune', '--omit=dev', '--no-save', '--no-audit', '--no-fund')
   await npm('install', '-w', 'docs', '--ignore-scripts', '--no-audit', '--no-fund')
-  await git.dirty()
+  if (isLocal && smokePublish) {
+    log.info(`Skipping git dirty check due to local smoke publish test being run`)
+  } else {
+    await git.dirty()
+  }
 
   for (const publish of publishes) {
     const workspace = publish.workspace && `--workspace=${publish.name}`
-    if (packOnly) {
+    const publishPkg = (...args) => npm('publish', workspace, `--tag=${publish.tag}`, ...args)
+    if (isPack) {
       await npm(
         'pack',
         workspace,
         opts.packDestination && `--pack-destination=${opts.packDestination}`
       )
+      if (smokePublish) {
+        await publishPkg('--dry-run')
+      }
     } else {
-      await npm(
-        'publish',
-        workspace,
-        `--tag=${publish.tag}`,
+      await publishPkg(
         opts.dryRun && '--dry-run',
         opts.otp && `--otp=${opts.otp === 'op' ? await op() : opts.otp}`
       )
diff --git a/scripts/smoke-publish-test.sh b/scripts/smoke-publish-test.sh
@@ -0,0 +1,92 @@
+ #!/usr/bin/env bash
+
+set -eo pipefail
+
+IS_LOCAL="false"
+IS_CI="true"
+
+if [ -z "$CI" ]; then
+  echo "Running locally will overwrite your globally installed npm."
+  GITHUB_SHA=$(git rev-parse HEAD)
+  RUNNER_TEMP=$(mktemp -d)
+  IS_LOCAL="true"
+  IS_CI="false"
+fi
+
+if [ -z "$GITHUB_SHA" ]; then
+  echo "Error: GITHUB_SHA is required"
+  exit 1
+fi
+
+if [ -z "$RUNNER_TEMP" ]; then
+  echo "Error: RUNNER_TEMP is required"
+  exit 1
+fi
+
+ORIGINAL_GLOBAL_NPM_VERSION=$(npm --version)
+if [ ${#ORIGINAL_GLOBAL_NPM_VERSION} -gt 40 ]; then
+  echo "Error: Global npm version already contains a git SHA ${ORIGINAL_GLOBAL_NPM_VERSION}"
+  exit 1
+fi
+
+ORIGINAL_LOCAL_NPM_VERSION=$(node . --version)
+if [ ${#ORIGINAL_LOCAL_NPM_VERSION} -gt 40 ]; then
+  echo "Error: Local npm version already contains a git SHA ${ORIGINAL_LOCAL_NPM_VERSION}"
+  exit 1
+fi
+NPM_VERSION="$ORIGINAL_LOCAL_NPM_VERSION-$GITHUB_SHA.0"
+
+# Only cleanup locally
+if [ "$IS_LOCAL" == "true" ]; then
+  function cleanup {
+    npm pkg set version=$ORIGINAL_LOCAL_NPM_VERSION
+    node scripts/resetdeps.js
+    if [ "$(git rev-parse HEAD)" != "$GITHUB_SHA" ]; then
+      echo "==================================="
+      echo "==================================="
+      echo "HEAD is on a different commit."
+      echo "==================================="
+      echo "==================================="
+    fi
+    if [ "$(npm --version)" == "$NPM_VERSION" ]; then
+      echo "==================================="
+      echo "==================================="
+      echo "Global npm version has changed to $NPM_VERSION"
+      echo "Run the following to change it back"
+      echo "npm install npm@$ORIGINAL_GLOBAL_NPM_VERSION -g"
+      echo "==================================="
+      echo "==================================="
+    fi
+  }
+  trap cleanup EXIT
+fi
+
+# Version the local source of npm with the current git sha and
+# and pack and install it globally the same way we would if we
+# were publishing it to the registry. The only difference is in the
+# publish.js script which will only pack and not publish
+node . version $NPM_VERSION --ignore-scripts --git-tag-version="$IS_CI"
+node scripts/publish.js --pack-destination=$RUNNER_TEMP --smoke-publish=true --is-local="$IS_LOCAL"
+NPM_TARBALL="$RUNNER_TEMP/npm-$NPM_VERSION.tgz"
+node . install --global $NPM_TARBALL
+
+# Only run the tests if we are sure we have the right version
+# otherwise the tests being run are pointless
+NPM_GLOBAL_VERSION="$(npm --version)"
+if [ "$NPM_GLOBAL_VERSION" != "$NPM_VERSION" ]; then
+  echo "global npm is not the correct version for smoke-publish"
+  echo "found: $NPM_GLOBAL_VERSION, expected: $NPM_VERSION"
+  exit 1
+fi
+
+# Install dev deps only for smoke tests so they can be run
+node . install -w smoke-tests --ignore-scripts --no-audit --no-fund
+# Run smoke tests with env vars so it uses the globally installed tarball we
+# just packed/installed. The tacked on args at the end are only used for
+# debugging locally when we want to pass args to the smoke-tests to limit the
+# files being run or grep a test, etc. Also now set CI=true so we get more
+# debug output in our tap tests
+CI="true" SMOKE_PUBLISH_NPM="1" SMOKE_PUBLISH_TARBALL="$NPM_TARBALL" npm test \
+  -w smoke-tests \
+  --ignore-scripts \
+  -- -Rtap "$@"
diff --git a/scripts/template-oss/ci-release-yml.hbs b/scripts/template-oss/ci-release-yml.hbs
@@ -11,27 +11,8 @@
       jobCreateCheck=(obj sha="${{ inputs.check-sha }}")
       windowsCI=false
     }}
-      - name: Pack
-        env:
-          SMOKE_PUBLISH_NPM: 1
-        run: |
-          NPM_VERSION="$({{ rootNpmPath }} --version)-$GITHUB_SHA.0"
-          {{ rootNpmPath }} version $NPM_VERSION --ignore-scripts
-          node scripts/publish.js --pack-destination=$RUNNER_TEMP
-          export SMOKE_PUBLISH_TARBALL="$RUNNER_TEMP/npm-$NPM_VERSION.tgz"
-          {{ rootNpmPath }} install --global $SMOKE_PUBLISH_TARBALL
-          {{ rootNpmPath }} install -w smoke-tests --ignore-scripts --no-audit --no-fund
-          # call installed npm instead of local source since we are testing
-          # the packed tarball that we just installed globally
-          NPM_GLOBAL_VERSION="$(npm --version)"
-          npm help
-          if [ "$NPM_GLOBAL_VERSION" == "$NPM_VERSION" ]; then
-            npm test -w smoke-tests --ignore-scripts
-          else
-            echo "global npm is not the correct version for smoke-publish"
-            echo "found: $NPM_GLOBAL_VERSION, expected: $NPM_VERSION"
-            exit 1
-          fi
+      - name: Smoke Publish
+        run: ./scripts/smoke-publish-test.sh
       - name: Conclude Check
         uses: LouisBrunner/checks-action@v1.6.0
         if: always()
diff --git a/scripts/util.js b/scripts/util.js
@@ -132,7 +132,7 @@ git.dirty = () => npmGit.isClean({ cwd: CWD }).then(async r => {
     return 'git clean'
   }
   await git('status', '--porcelain=v1', '-uno')
-  await git('diff')
+  await git('--no-pager', 'diff')
   throw new Error('git dirty')
 })
 
diff --git a/smoke-tests/test/fixtures/setup.js b/smoke-tests/test/fixtures/setup.js
@@ -75,6 +75,8 @@ const getCleanPaths = async () => {
 
 module.exports = async (t, { testdir = {}, debug, mockRegistry = true, useProxy = false } = {}) => {
   const debugLog = debug || CI ? (...a) => t.comment(...a) : () => {}
+  debugLog({ SMOKE_PUBLISH_NPM, SMOKE_PUBLISH_TARBALL, CI })
+
   const cleanPaths = await getCleanPaths()
 
   // setup fixtures
diff --git a/smoke-tests/test/npm-replace-global.js b/smoke-tests/test/npm-replace-global.js
@@ -94,7 +94,6 @@ t.test('pack and replace global self', async t => {
 })
 
 t.test('publish and replace global self', async t => {
-  let publishedPackument = null
   const pkg = require('../../package.json')
   const { name, version } = pkg
 
@@ -114,39 +113,44 @@ t.test('publish and replace global self', async t => {
     },
   })
 
-  const npmPackage = async ({ manifest, ...opts } = {}) => {
+  const mockNpmPackage = async ({ manifest, ...opts } = {}) => {
     await registry.package({
       manifest: registry.manifest({ name, ...manifest }),
       ...opts,
     })
+    await fs.rm(cache, { recursive: true, force: true })
   }
 
-  const npmInstall = async (useNpm) => {
-    await npmPackage({
-      manifest: { packuments: [publishedPackument] },
-      tarballs: { [version]: tarball },
-      times: 3,
-    })
-    await fs.rm(cache, { recursive: true, force: true })
-    await useNpm('install', 'npm@latest', '--global')
-    return getPaths()
+  const mockNpmPublish = () => {
+    let publishedPackument = null
+    registry.nock.put('/npm', body => {
+      if (body._id === 'npm' && body.versions[version]) {
+        publishedPackument = body.versions[version]
+        return true
+      }
+      return false
+    }).reply(201, {})
+    return {
+      get packument () {
+        return publishedPackument
+      },
+    }
   }
 
+  const publish = mockNpmPublish()
+  await mockNpmPackage()
+  await npmLocal('publish', { proxy: true, force: true })
+
   const tarball = await npmLocalTarball()
 
-  if (setup.SMOKE_PUBLISH) {
-    await npmPackage()
-  }
-  registry.nock.put('/npm', body => {
-    if (body._id === 'npm' && body.versions[version]) {
-      publishedPackument = body.versions[version]
-      return true
-    }
-    return false
-  }).reply(201, {})
-  await npmLocal('publish', { proxy: true, force: true })
+  await mockNpmPackage({
+    manifest: { packuments: [publish.packument] },
+    tarballs: { [version]: tarball },
+    times: 3,
+  })
+  await npm('install', 'npm@latest', '--global')
+  const paths = await getPaths()
 
-  const paths = await npmInstall(npm)
   t.equal(paths.npmRoot, join(globalNodeModules, 'npm'), 'npm root is in the testdir')
   t.equal(paths.pathNpm, join(globalBin, 'npm'), 'npm bin is in the testdir')
   t.equal(paths.pathNpx, join(globalBin, 'npx'), 'npx bin is in the testdir')
@@ -159,7 +163,13 @@ t.test('publish and replace global self', async t => {
     'bin has npm and npx'
   )
 
-  t.strictSame(await npmInstall(npmPath), paths)
+  await mockNpmPackage({
+    manifest: { packuments: [publish.packument] },
+    tarballs: { [version]: tarball },
+    times: 3,
+  })
+  await npmPath('install', 'npm@latest', '--global')
+  t.strictSame(await getPaths(), paths)
 })
 
 t.test('fail when updating with lazy require', async t => {

Original file line number	Diff line number	Diff line change
`@@ -132,7 +132,7 @@ git.dirty = () => npmGit.isClean({ cwd: CWD }).then(async r => {`
`132`	`132`	`return 'git clean'`
`133`	`133`	`}`
`134`	`134`	`await git('status', '--porcelain=v1', '-uno')`
`135`		`- await git('diff')`
	`135`	`+ await git('--no-pager', 'diff')`
`136`	`136`	`throw new Error('git dirty')`
`137`	`137`	`})`
`138`	`138`