From c4eefc3be19aec951b7c90b38aa71ee9f3e96789 Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Fri, 25 Jun 2021 09:59:04 +0200 Subject: [PATCH 1/8] Run rustdoc-gui tests in parallel --- src/tools/rustdoc-gui/tester.js | 37 +++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index 416d824c5645e..491884b078127 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -70,6 +70,14 @@ function parseOptions(args) { return null; } +function print_test_successful() { + process.stdout.write("."); +} +function print_test_erroneous() { + // Bold Red "F" Reset + process.stdout.write("\033[1m\x1b[31mF\x1b[0m"); +} + async function main(argv) { let opts = parseOptions(argv.slice(2)); if (opts === null) { @@ -100,27 +108,48 @@ async function main(argv) { let failed = false; let files; + let tests = []; if (opts["files"].length === 0) { files = fs.readdirSync(opts["tests_folder"]).filter(file => path.extname(file) == ".goml"); } else { files = opts["files"].filter(file => path.extname(file) == ".goml"); } - + if (files.length === 0) { + console.log("rustdoc-gui: No test selected"); + process.exit(2); + } files.sort(); + + console.log(`running ${files.length} rustdoc-gui tests`); + process.setMaxListeners(files.length + 1); for (var i = 0; i < files.length; ++i) { const testPath = path.join(opts["tests_folder"], files[i]); - await runTest(testPath, options).then(out => { + tests.push(runTest(testPath, options)); + } + + let error_outputs = ""; + let failed_outputs = ""; + for (var i = 0; i < tests.length; ++i) { + await tests[i].then(out => { const [output, nb_failures] = out; - console.log(output); if (nb_failures > 0) { + failed_outputs += output + "\n"; + print_test_erroneous() failed = true; + } else { + print_test_successful() } }).catch(err => { - console.error(err); + error_outputs += err + "\n"; + print_test_erroneous(); failed = true; }); } + console.log("") + if (failed) { + console.log(failed_outputs); + console.error(error_outputs); process.exit(1); } } From a1daa8d984eb715c87ca010c8fd0ac96a698d214 Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Tue, 29 Jun 2021 01:01:40 +0200 Subject: [PATCH 2/8] Capture outputs and show output when debug flag is passed --- src/tools/rustdoc-gui/tester.js | 100 +++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 28 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index 491884b078127..ca11c286ad373 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -73,9 +73,9 @@ function parseOptions(args) { function print_test_successful() { process.stdout.write("."); } + function print_test_erroneous() { - // Bold Red "F" Reset - process.stdout.write("\033[1m\x1b[31mF\x1b[0m"); + process.stdout.write("F"); } async function main(argv) { @@ -84,6 +84,7 @@ async function main(argv) { process.exit(1); } + let debug = false; const options = new Options(); try { // This is more convenient that setting fields one by one. @@ -92,6 +93,7 @@ async function main(argv) { "--variable", "DOC_PATH", opts["doc_folder"], ]; if (opts["debug"]) { + debug = true; args.push("--debug"); } if (opts["show_text"]) { @@ -108,12 +110,12 @@ async function main(argv) { let failed = false; let files; - let tests = []; if (opts["files"].length === 0) { - files = fs.readdirSync(opts["tests_folder"]).filter(file => path.extname(file) == ".goml"); + files = fs.readdirSync(opts["tests_folder"]); } else { - files = opts["files"].filter(file => path.extname(file) == ".goml"); + files = opts["files"]; } + files = files.filter(file => path.extname(file) == ".goml"); if (files.length === 0) { console.log("rustdoc-gui: No test selected"); process.exit(2); @@ -122,34 +124,76 @@ async function main(argv) { console.log(`running ${files.length} rustdoc-gui tests`); process.setMaxListeners(files.length + 1); - for (var i = 0; i < files.length; ++i) { + let tests = []; + let results = new Array(files.length); + // poormans enum + const RUN_SUCCESS = 42, RUN_FAILED = 23, RUN_ERRORED = 13; + for (let i = 0; i < files.length; ++i) { const testPath = path.join(opts["tests_folder"], files[i]); - tests.push(runTest(testPath, options)); - } - - let error_outputs = ""; - let failed_outputs = ""; - for (var i = 0; i < tests.length; ++i) { - await tests[i].then(out => { - const [output, nb_failures] = out; - if (nb_failures > 0) { - failed_outputs += output + "\n"; - print_test_erroneous() + tests.push( + runTest(testPath, options) + .then(out => { + //console.log(i); + const [output, nb_failures] = out; + results[i] = { + status: nb_failures === 0 ? RUN_SUCCESS : RUN_FAILED, + output: output, + }; + if (nb_failures > 0) { + print_test_erroneous() + failed = true; + } else { + print_test_successful() + } + }) + .catch(err => { + results[i] = { + status: RUN_ERRORED, + output: err, + }; + print_test_erroneous(); failed = true; - } else { - print_test_successful() - } - }).catch(err => { - error_outputs += err + "\n"; - print_test_erroneous(); - failed = true; - }); + }) + ); } - console.log("") + await Promise.all(tests); + // final \n after the tests + console.log("\n"); + + results.forEach(r => { + switch (r.status) { + case RUN_SUCCESS: + if (debug === false) { + break; + } + case RUN_FAILED: + console.log(r.output); + break; + case RUN_ERRORED: + // skip + break; + default: + console.error(`unexpected status = ${r.status}`); + process.exit(4); + } + }); + // print run errors on the bottom so developers see them better + results.forEach(r => { + switch (r.status) { + case RUN_SUCCESS: + case RUN_FAILED: + // skip + break; + case RUN_ERRORED: + console.error(r.output); + break; + default: + console.error(`unexpected status = ${r.status}`); + process.exit(4); + } + }); if (failed) { - console.log(failed_outputs); - console.error(error_outputs); process.exit(1); } } From 40136e196992b3861b938a44cbc90780b3f4aca7 Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Tue, 29 Jun 2021 01:41:39 +0200 Subject: [PATCH 3/8] Implement sequential execution --- src/tools/rustdoc-gui/tester.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index ca11c286ad373..69665693104a9 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -84,7 +84,10 @@ async function main(argv) { process.exit(1); } + // Print successful tests too let debug = false; + // Run tests in sequentially + let no_headless = false; const options = new Options(); try { // This is more convenient that setting fields one by one. @@ -101,6 +104,7 @@ async function main(argv) { } if (opts["no_headless"]) { args.push("--no-headless"); + no_headless = true; } options.parseArguments(args); } catch (error) { @@ -155,6 +159,9 @@ async function main(argv) { failed = true; }) ); + if (no_headless) { + await tests[i]; + } } await Promise.all(tests); // final \n after the tests @@ -187,9 +194,6 @@ async function main(argv) { case RUN_ERRORED: console.error(r.output); break; - default: - console.error(`unexpected status = ${r.status}`); - process.exit(4); } }); From e540aa08b4ed2a44b07c911287d392a9f3a068a2 Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Thu, 1 Jul 2021 16:30:28 +0200 Subject: [PATCH 4/8] Review by @GuillaumeGomez Co-authored-by: Guillaume Gomez --- src/tools/rustdoc-gui/tester.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index 69665693104a9..3f83aa425dffd 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -121,12 +121,12 @@ async function main(argv) { } files = files.filter(file => path.extname(file) == ".goml"); if (files.length === 0) { - console.log("rustdoc-gui: No test selected"); + console.error("rustdoc-gui: No test selected"); process.exit(2); } files.sort(); - console.log(`running ${files.length} rustdoc-gui tests`); + console.log(`Running ${files.length} rustdoc-gui tests...`); process.setMaxListeners(files.length + 1); let tests = []; let results = new Array(files.length); @@ -137,7 +137,6 @@ async function main(argv) { tests.push( runTest(testPath, options) .then(out => { - //console.log(i); const [output, nb_failures] = out; results[i] = { status: nb_failures === 0 ? RUN_SUCCESS : RUN_FAILED, @@ -186,14 +185,8 @@ async function main(argv) { }); // print run errors on the bottom so developers see them better results.forEach(r => { - switch (r.status) { - case RUN_SUCCESS: - case RUN_FAILED: - // skip - break; - case RUN_ERRORED: - console.error(r.output); - break; + if (r.status === RUN_ERRORED) { + console.error(r.output); } }); From eca2b3be3d7d52cb03277fb86e57cf3bc969694a Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Thu, 1 Jul 2021 16:46:14 +0200 Subject: [PATCH 5/8] Switch to stderr for failed tests --- src/tools/rustdoc-gui/tester.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index 3f83aa425dffd..f0e5aef0ec15f 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -75,7 +75,7 @@ function print_test_successful() { } function print_test_erroneous() { - process.stdout.write("F"); + process.stderr.write("F"); } async function main(argv) { From c17628be18b1c99b1cb4c78b28670068da0f2fc4 Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Mon, 9 Aug 2021 22:34:44 +0200 Subject: [PATCH 6/8] Implement summarised results --- src/tools/rustdoc-gui/tester.js | 91 +++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index f0e5aef0ec15f..92535f3417839 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -70,12 +70,33 @@ function parseOptions(args) { return null; } -function print_test_successful() { - process.stdout.write("."); +/// Print single char status information without \n +function char_printer(n_tests) { + const max_per_line = 10; + let current = 0; + return { + successful: function() { + current += 1; + if (current % max_per_line === 0) { + process.stdout.write(`. (${current}/${n_tests})\n`); + } else { + process.stdout.write("."); + } + }, + erroneous: function() { + current += 1; + if (current % max_per_line === 0) { + process.stderr.write(`F (${current}/${n_tests})\n`); + } else { + process.stderr.write("F"); + } + }, + }; } -function print_test_erroneous() { - process.stderr.write("F"); +/// Sort array by .file_name property +function by_filename(a, b) { + return a.file_name - b.file_name; } async function main(argv) { @@ -129,32 +150,36 @@ async function main(argv) { console.log(`Running ${files.length} rustdoc-gui tests...`); process.setMaxListeners(files.length + 1); let tests = []; - let results = new Array(files.length); - // poormans enum - const RUN_SUCCESS = 42, RUN_FAILED = 23, RUN_ERRORED = 13; + let results = { + successful: [], + failed: [], + errored: [], + }; + const status_bar = char_printer(files.length); for (let i = 0; i < files.length; ++i) { - const testPath = path.join(opts["tests_folder"], files[i]); + const file_name = files[i]; + const testPath = path.join(opts["tests_folder"], file_name); tests.push( runTest(testPath, options) .then(out => { const [output, nb_failures] = out; - results[i] = { - status: nb_failures === 0 ? RUN_SUCCESS : RUN_FAILED, + results[nb_failures === 0 ? "successful" : "failed"].push({ + file_name: file_name, output: output, - }; + }); if (nb_failures > 0) { - print_test_erroneous() + status_bar.erroneous() failed = true; } else { - print_test_successful() + status_bar.successful() } }) .catch(err => { - results[i] = { - status: RUN_ERRORED, + results.errored.push({ + file_name: file_name, output: err, - }; - print_test_erroneous(); + }); + status_bar.erroneous(); failed = true; }) ); @@ -166,28 +191,20 @@ async function main(argv) { // final \n after the tests console.log("\n"); - results.forEach(r => { - switch (r.status) { - case RUN_SUCCESS: - if (debug === false) { - break; - } - case RUN_FAILED: - console.log(r.output); - break; - case RUN_ERRORED: - // skip - break; - default: - console.error(`unexpected status = ${r.status}`); - process.exit(4); - } + if (debug === false) { + results.successful.sort(by_filename); + results.successful.forEach(r => { + console.log(r.output); + }); + } + results.failed.sort(by_filename); + results.failed.forEach(r => { + console.log(r.output); }); // print run errors on the bottom so developers see them better - results.forEach(r => { - if (r.status === RUN_ERRORED) { - console.error(r.output); - } + results.errored.sort(by_filename); + results.errored.forEach(r => { + console.error(r.output); }); if (failed) { From 86fa21cda57b7235e191926012ffa6e3b534e92b Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Sat, 14 Aug 2021 23:19:28 +0200 Subject: [PATCH 7/8] Implement a finish method for the status_bar and some cleanup --- src/tools/rustdoc-gui/tester.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index 92535f3417839..fcecd00bd5370 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -5,6 +5,7 @@ // ``` const fs = require("fs"); const path = require("path"); +const os = require('os'); const {Options, runTest} = require('browser-ui-test'); function showHelp() { @@ -78,7 +79,7 @@ function char_printer(n_tests) { successful: function() { current += 1; if (current % max_per_line === 0) { - process.stdout.write(`. (${current}/${n_tests})\n`); + process.stdout.write(`. (${current}/${n_tests})${os.EOL}`); } else { process.stdout.write("."); } @@ -86,11 +87,15 @@ function char_printer(n_tests) { erroneous: function() { current += 1; if (current % max_per_line === 0) { - process.stderr.write(`F (${current}/${n_tests})\n`); + process.stderr.write(`F (${current}/${n_tests})${os.EOL}`); } else { process.stderr.write("F"); } }, + finish: function() { + const spaces = " ".repeat(max_per_line - (current % max_per_line)); + process.stdout.write(`${spaces} (${current}/${n_tests})${os.EOL}${os.EOL}`); + }, }; } @@ -187,9 +192,10 @@ async function main(argv) { await tests[i]; } } - await Promise.all(tests); - // final \n after the tests - console.log("\n"); + if (!no_headless) { + await Promise.all(tests); + } + status_bar.finish(); if (debug === false) { results.successful.sort(by_filename); From 7f2b52b1d6b7d8bc1accf5dbb99d5281dc69ae07 Mon Sep 17 00:00:00 2001 From: Stefan Schindler Date: Sat, 14 Aug 2021 23:36:17 +0200 Subject: [PATCH 8/8] More spacing between the different blocks of results --- src/tools/rustdoc-gui/tester.js | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index fcecd00bd5370..4e77d27d399c2 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -197,21 +197,28 @@ async function main(argv) { } status_bar.finish(); - if (debug === false) { + if (debug) { results.successful.sort(by_filename); results.successful.forEach(r => { console.log(r.output); }); } - results.failed.sort(by_filename); - results.failed.forEach(r => { - console.log(r.output); - }); - // print run errors on the bottom so developers see them better - results.errored.sort(by_filename); - results.errored.forEach(r => { - console.error(r.output); - }); + + if (results.failed.length > 0) { + console.log(""); + results.failed.sort(by_filename); + results.failed.forEach(r => { + console.log(r.output); + }); + } + if (results.errored.length > 0) { + console.log(os.EOL); + // print run errors on the bottom so developers see them better + results.errored.sort(by_filename); + results.errored.forEach(r => { + console.error(r.output); + }); + } if (failed) { process.exit(1);