diff --git a/scripts/generate-clients/code-gen-dir.js b/scripts/generate-clients/code-gen-dir.js index b4603978249f..64bee1ab1972 100644 --- a/scripts/generate-clients/code-gen-dir.js +++ b/scripts/generate-clients/code-gen-dir.js @@ -8,6 +8,7 @@ const CODE_GEN_SDK_ROOT = getCodeGenDirRoot("sdk-codegen"); const CODE_GEN_PROTOCOL_TESTS_ROOT = getCodeGenDirRoot("protocol-test-codegen"); const TEMP_CODE_GEN_INPUT_DIR = normalize(join(__dirname, ".aws-models")); +const DEFAULT_CODE_GEN_INPUT_DIR = normalize(join(CODE_GEN_SDK_ROOT, "aws-models")); const getCodeGenOutputDir = (dir) => normalize(join(__dirname, "..", "..", "codegen", dir, "build", "smithyprojections", dir)); @@ -22,5 +23,6 @@ module.exports = { CODE_GEN_SDK_OUTPUT_DIR, CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR, CODE_GEN_GENERIC_CLIENT_OUTPUT_DIR, + DEFAULT_CODE_GEN_INPUT_DIR, TEMP_CODE_GEN_INPUT_DIR, }; diff --git a/scripts/generate-clients/code-gen.js b/scripts/generate-clients/code-gen.js index 1d7609061732..eed25153d427 100644 --- a/scripts/generate-clients/code-gen.js +++ b/scripts/generate-clients/code-gen.js @@ -1,51 +1,28 @@ // @ts-check -const path = require("path"); +const { basename, join, relative } = require("path"); const { emptyDirSync } = require("fs-extra"); -const { copyFileSync, readdirSync, lstatSync } = require("fs"); +const { copyFileSync } = require("fs"); const { spawnProcess } = require("../utils/spawn-process"); const { CODE_GEN_ROOT, CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR } = require("./code-gen-dir"); -const Glob = require("glob"); +const { getModelFilepaths } = require("./get-model-filepaths"); -const generateClients = async (models) => { - let designatedModels = false; - if (typeof models === "string") { - //`models` is a folder path - designatedModels = true; +const generateClients = async (models, batchSize) => { + const filepaths = getModelFilepaths(models); + const options = [ + ":sdk-codegen:clean", + ":sdk-codegen:build", + `-PmodelsDirProp=${relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR)}`, + ]; + + while (filepaths.length > 0) { emptyDirSync(TEMP_CODE_GEN_INPUT_DIR); - console.log(`preparing models from ${models}...`); - for (const modelFileName of readdirSync(models)) { - const modelPath = path.join(models, modelFileName); - if (!lstatSync(modelPath).isFile()) continue; - console.log(`copying model ${modelFileName}...`); - copyFileSync(modelPath, path.join(TEMP_CODE_GEN_INPUT_DIR, modelFileName)); + const filepathsToCopy = filepaths.splice(0, batchSize); + for (const filepath of filepathsToCopy) { + const filename = basename(filepath); + copyFileSync(filepath, join(TEMP_CODE_GEN_INPUT_DIR, filename)); } - } else if (Array.isArray(models)) { - //`models` is a list of globs - designatedModels = true; - emptyDirSync(TEMP_CODE_GEN_INPUT_DIR); - models.forEach((pattern) => { - const files = Glob.sync(pattern, { - realpath: true, - absolute: true, - }); - files.forEach((file) => { - if (!lstatSync(file).isFile()) return; - const name = path.basename(file); - console.log(`copying model ${name}...`); - copyFileSync(file, path.join(TEMP_CODE_GEN_INPUT_DIR, name)); - }); - }); - } else { - console.log("no model supplied, generating all AWS clients"); - } - const options = [":sdk-codegen:clean", ":sdk-codegen:build"]; - if (designatedModels) { - options.push(`-PmodelsDirProp=${path.relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR)}`); + await spawnProcess("./gradlew", options, { cwd: CODE_GEN_ROOT }); } - - await spawnProcess("./gradlew", options, { - cwd: CODE_GEN_ROOT, - }); }; const generateProtocolTests = async () => { diff --git a/scripts/generate-clients/get-model-filepaths.js b/scripts/generate-clients/get-model-filepaths.js new file mode 100644 index 000000000000..e95667e3c62a --- /dev/null +++ b/scripts/generate-clients/get-model-filepaths.js @@ -0,0 +1,27 @@ +const { readdirSync, lstatSync } = require("fs"); +const Glob = require("glob"); +const { join } = require("path"); + +const getModelFilepaths = (models) => { + if (typeof models === "string") { + //`models` is a folder path + console.log(`preparing models from ${models}...`); + return readdirSync(models) + .map((modelFileName) => join(models, modelFileName)) + .filter((modelFilepath) => lstatSync(modelFilepath).isFile()); + } else if (Array.isArray(models)) { + //`models` is a list of globs + console.log(`preparing models from ${models}...`); + return models + .map((pattern) => + Glob.sync(pattern, { + realpath: true, + absolute: true, + }) + ) + .flat() + .filter((modelFilepath) => lstatSync(modelFilepath).isFile()); + } +}; + +module.exports = { getModelFilepaths }; diff --git a/scripts/generate-clients/index.js b/scripts/generate-clients/index.js index 6193548cae4b..d4df9af93f11 100644 --- a/scripts/generate-clients/index.js +++ b/scripts/generate-clients/index.js @@ -8,6 +8,7 @@ const { CODE_GEN_SDK_OUTPUT_DIR, CODE_GEN_GENERIC_CLIENT_OUTPUT_DIR, CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR, + DEFAULT_CODE_GEN_INPUT_DIR, TEMP_CODE_GEN_INPUT_DIR, } = require("./code-gen-dir"); const { prettifyCode } = require("./code-prettify"); @@ -22,6 +23,7 @@ const { output: clientsDir, noPrivateClients, s: serverOnly, + batchSize, } = yargs .alias("m", "models") .string("m") @@ -41,6 +43,10 @@ const { .boolean("s") .describe("s", "Generate server artifacts instead of client ones") .conflicts("s", ["m", "g", "n"]) + .describe("b", "Batchsize for generating clients") + .number("b") + .alias("b", "batch-size") + .default("b", 50) .help().argv; (async () => { @@ -58,7 +64,7 @@ const { return; } - await generateClients(models || globs); + await generateClients(models || globs || DEFAULT_CODE_GEN_INPUT_DIR, batchSize); if (!noPrivateClients) { await generateGenericClient(); await generateProtocolTests();