Skip to content

Commit 0362f85

Browse files
authored
Turbopack build: Add devlow-bench (#70511)
Ensures Webpack build and Turbopack build results for the heavy-npm-deps benchmark are uploaded to DataDog so that we can track them over time. <!-- Thanks for opening a PR! Your contribution is much appreciated. To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below. Choose the right checklist for the change(s) that you're making: ## For Contributors ### Improving Documentation - Run `pnpm prettier-fix` to fix formatting issues before opening the PR. - Read the Docs Contribution Guide to ensure your contribution follows the docs guidelines: https://nextjs.org/docs/community/contribution-guide ### Adding or Updating Examples - The "examples guidelines" are followed from our contributing doc https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md - Make sure the linting passes by running `pnpm build && pnpm lint`. See https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md ### Fixing a bug - Related issues linked using `fixes #number` - Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs - Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md ### Adding a feature - Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas) - Related issues/discussions are linked using `fixes #number` - e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs) - Documentation added - Telemetry added. In case of a feature if it's used or not. - Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md ## For Maintainers - Minimal description (aim for explaining to someone not on the team to understand the PR) - When linking to a Slack thread, you might want to share details of the conclusion - Link both the Linear (Fixes NEXT-xxx) and the GitHub issues - Add review comments if necessary to explain to the reviewer the logic behind a change ### What? ### Why? ### How? Closes NEXT- Fixes # -->
1 parent 334f335 commit 0362f85

File tree

3 files changed

+229
-8
lines changed

3 files changed

+229
-8
lines changed

.github/workflows/build_and_test.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,8 @@ jobs:
186186
- '--turbopack=false'
187187
- '--turbopack=true'
188188
selector:
189-
- '--scenario=heavy-npm-deps --page=homepage'
189+
- '--scenario=heavy-npm-deps-dev --page=homepage'
190+
- '--scenario=heavy-npm-deps-build --page=homepage'
190191
uses: ./.github/workflows/build_reusable.yml
191192
with:
192193
afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }}

bench/heavy-npm-deps/package.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
"scripts": {
66
"dev-turbopack": "next dev --turbo",
77
"dev-webpack": "next dev",
8+
"build-turbopack": "TURBOPACK=1 TURBOPACK_BUILD=1 next build",
9+
"build-webpack": "next build",
10+
"start-turbopack": "TURBOPACK=1 next start",
11+
"start-webpack": "next start",
812
"build-application": "next build",
913
"start-application": "next start"
1014
},

scripts/devlow-bench.mjs

Lines changed: 223 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,212 @@ const GIT_BRANCH =
2525
return cmd.output
2626
})())
2727

28+
const nextBuildWorkflow =
29+
(benchmarkName, pages) =>
30+
async ({ turbopack, page }) => {
31+
const pageConfig =
32+
typeof pages[page] === 'string' ? { url: pages[page] } : pages[page]
33+
const cleanupTasks = []
34+
try {
35+
const env = {
36+
PATH: process.env.PATH,
37+
NODE: process.env.NODE,
38+
HOSTNAME: process.env.HOSTNAME,
39+
PWD: process.env.PWD,
40+
// Disable otel initialization to prevent pending / hanging request to otel collector
41+
OTEL_SDK_DISABLED: 'true',
42+
NEXT_PUBLIC_OTEL_SENTRY: 'true',
43+
NEXT_PUBLIC_OTEL_DEV_DISABLED: 'true',
44+
NEXT_TRACE_UPLOAD_DISABLED: 'true',
45+
// Enable next.js test mode to get HMR events
46+
__NEXT_TEST_MODE: '1',
47+
}
48+
49+
const benchmarkDir = resolve(REPO_ROOT, 'bench', benchmarkName)
50+
51+
// cleanup .next directory to remove persistent cache
52+
await retry(() =>
53+
rm(join(benchmarkDir, '.next'), { recursive: true, force: true })
54+
)
55+
56+
await measureTime('cleanup', {
57+
scenario: benchmarkName,
58+
props: { turbopack, page },
59+
})
60+
61+
const buildArgs = [turbopack ? 'build-turbopack' : 'build-webpack']
62+
let buildShell = command('pnpm', buildArgs, {
63+
cwd: benchmarkDir,
64+
env,
65+
})
66+
await buildShell.ok()
67+
68+
await measureTime('build', {
69+
scenario: benchmarkName,
70+
props: { turbopack, page },
71+
})
72+
73+
// startup browser
74+
let session = await newBrowserSession({})
75+
const closeSession = async () => {
76+
if (session) {
77+
await session.close()
78+
session = null
79+
}
80+
}
81+
cleanupTasks.push(closeSession)
82+
await measureTime('browser startup', {
83+
props: { turbopack, page },
84+
})
85+
86+
// run command to start dev server
87+
const startArgs = [turbopack ? 'start-turbopack' : 'start-webpack']
88+
let shell = command('pnpm', startArgs, {
89+
cwd: benchmarkDir,
90+
env,
91+
})
92+
const killShell = async () => {
93+
if (shell) {
94+
await shell.kill()
95+
shell = null
96+
}
97+
}
98+
cleanupTasks.push(killShell)
99+
100+
// wait for server to be ready
101+
const START_SERVER_REGEXP = /Local:\s+(?<url>.+)\n/
102+
const {
103+
groups: { url },
104+
} = await shell.waitForOutput(START_SERVER_REGEXP)
105+
await measureTime('server startup', { props: { turbopack, page } })
106+
await shell.reportMemUsage('mem usage after startup', {
107+
props: { turbopack, page },
108+
})
109+
110+
// open page
111+
const pageInstance = await session.hardNavigation(
112+
'open page',
113+
url + pageConfig.url
114+
)
115+
await shell.reportMemUsage('mem usage after open page')
116+
117+
let status = 0
118+
try {
119+
if (
120+
await pageInstance.evaluate(
121+
'!next.appDir && __NEXT_DATA__.page === "/404"'
122+
)
123+
) {
124+
status = 2
125+
}
126+
} catch (e) {
127+
status = 2
128+
}
129+
130+
try {
131+
if (
132+
!(await pageInstance.evaluate(
133+
'next.appDir || __NEXT_DATA__.page && !__NEXT_DATA__.err'
134+
))
135+
) {
136+
status = 1
137+
}
138+
} catch (e) {
139+
status = 1
140+
}
141+
142+
await reportMeasurement('page status', status, 'status code')
143+
144+
// reload page
145+
await session.reload('reload page')
146+
147+
await reportMeasurement(
148+
'console output',
149+
shell.output.split(/\n/).length,
150+
'lines'
151+
)
152+
153+
if (turbopack) {
154+
// close dev server and browser
155+
await killShell()
156+
await closeSession()
157+
} else {
158+
// wait for persistent cache to be written
159+
const waitPromise = new Promise((resolve) => {
160+
setTimeout(resolve, 5000)
161+
})
162+
const cacheLocation = join(
163+
benchmarkDir,
164+
'.next',
165+
'cache',
166+
'webpack',
167+
'client-development'
168+
)
169+
await Promise.race([
170+
waitForFile(join(cacheLocation, 'index.pack')),
171+
waitForFile(join(cacheLocation, 'index.pack.gz')),
172+
])
173+
await measureTime('cache created')
174+
await waitPromise
175+
await measureTime('waiting')
176+
177+
// close dev server and browser
178+
await killShell()
179+
await closeSession()
180+
}
181+
182+
buildShell = command('pnpm', buildArgs, {
183+
cwd: benchmarkDir,
184+
env,
185+
})
186+
await buildShell.ok()
187+
188+
await measureTime('build with cache', {
189+
scenario: benchmarkName,
190+
props: { turbopack, page },
191+
})
192+
193+
// startup new browser
194+
session = await newBrowserSession({})
195+
await measureTime('browser startup', {
196+
props: { turbopack, page },
197+
})
198+
199+
// run command to start dev server
200+
shell = command('pnpm', startArgs, {
201+
cwd: benchmarkDir,
202+
env,
203+
})
204+
205+
// wait for server to be ready
206+
const {
207+
groups: { url: url2 },
208+
} = await shell.waitForOutput(START_SERVER_REGEXP)
209+
await shell.reportMemUsage('mem usage after startup with cache')
210+
211+
// open page
212+
await session.hardNavigation(
213+
'open page with cache',
214+
url2 + pageConfig.url
215+
)
216+
217+
await reportMeasurement(
218+
'console output with cache',
219+
shell.output.split(/\n/).length,
220+
'lines'
221+
)
222+
await shell.reportMemUsage('mem usage after open page with cache')
223+
} catch (e) {
224+
console.log('CAUGHT', e)
225+
throw e
226+
} finally {
227+
// This must run in order
228+
// eslint-disable-next-line no-await-in-loop
229+
for (const task of cleanupTasks.reverse()) await task()
230+
await measureTime('shutdown')
231+
}
232+
}
233+
28234
const nextDevWorkflow =
29235
(benchmarkName, pages) =>
30236
async ({ turbopack, page }) => {
@@ -41,7 +247,7 @@ const nextDevWorkflow =
41247

42248
await measureTime('cleanup', {
43249
scenario: benchmarkName,
44-
props: { turbopack: null, page: null },
250+
props: { turbopack, page },
45251
})
46252

47253
// startup browser
@@ -54,15 +260,14 @@ const nextDevWorkflow =
54260
}
55261
cleanupTasks.push(closeSession)
56262
await measureTime('browser startup', {
57-
props: { turbopack: null, page: null },
263+
props: { turbopack, page },
58264
})
59265

60266
const env = {
61267
PATH: process.env.PATH,
62268
NODE: process.env.NODE,
63269
HOSTNAME: process.env.HOSTNAME,
64270
PWD: process.env.PWD,
65-
NODE_ENV: 'development',
66271
// Disable otel initialization to prevent pending / hanging request to otel collector
67272
OTEL_SDK_DISABLED: 'true',
68273
NEXT_PUBLIC_OTEL_SENTRY: 'true',
@@ -91,9 +296,9 @@ const nextDevWorkflow =
91296
const {
92297
groups: { url },
93298
} = await shell.waitForOutput(START_SERVER_REGEXP)
94-
await measureTime('server startup', { props: { page: null } })
299+
await measureTime('server startup', { props: { turbopack, page } })
95300
await shell.reportMemUsage('mem usage after startup', {
96-
props: { page: null },
301+
props: { turbopack, page },
97302
})
98303

99304
// open page
@@ -304,7 +509,7 @@ const nextDevWorkflow =
304509
// startup new browser
305510
session = await newBrowserSession({})
306511
await measureTime('browser startup', {
307-
props: { turbopack: null, page: null },
512+
props: { turbopack, page },
308513
})
309514

310515
// run command to start dev server
@@ -353,14 +558,25 @@ const pages = {
353558
}
354559

355560
describe(
356-
'heavy-npm-deps dev test',
561+
'heavy-npm-deps-dev',
357562
{
358563
turbopack: true,
564+
mode: 'dev',
359565
page: Object.keys(pages),
360566
},
361567
nextDevWorkflow('heavy-npm-deps', pages)
362568
)
363569

570+
describe(
571+
'heavy-npm-deps-build',
572+
{
573+
turbopack: true,
574+
mode: 'build',
575+
page: Object.keys(pages),
576+
},
577+
nextBuildWorkflow('heavy-npm-deps', pages)
578+
)
579+
364580
async function retry(fn) {
365581
let lastError
366582
for (let i = 100; i < 2000; i += 100) {

0 commit comments

Comments
 (0)