feat(pipelines): step outputs (#19024)

Make it possible to export environment variables from a CodeBuildStep,
and pipeline sources, and use them in the environment variables of
a CodeBuildStep or ShellStep.

Closes #17189, closes #18893, closes #15943, closes #16407.


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/@aws-cdk/pipelines/README.md

@@ -338,6 +338,40 @@ const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {
 
 You can adapt these examples to your own situation.
 
+#### Migrating from buildspec.yml files
+
+You may currently have the build instructions for your CodeBuild Projects in a
+`buildspec.yml` file in your source repository. In addition to your build
+commands, the CodeBuild Project's buildspec also controls some information that
+CDK Pipelines manages for you, like artifact identifiers, input artifact
+locations, Docker authorization, and exported variables.
+
+Since there is no way in general for CDK Pipelines to modify the file in your
+resource repository, CDK Pipelines configures the BuildSpec directly on the
+CodeBuild Project, instead of loading it from the `buildspec.yml` file.
+This requires a pipeline self-mutation to update.
+
+To avoid this, put your build instructions in a separate script, for example
+`build.sh`, and call that script from the build `commands` array:
+
+```ts
+declare const source: pipelines.IFileSetProducer;
+
+const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {
+  synth: new pipelines.ShellStep('Synth', {
+    input: source,
+    commands: [
+      // Abstract over doing the build
+      './build.sh',
+    ],
+  })
+});
+```
+
+Doing so keeps your exact build instructions in sync with your source code in
+the source repository where it belongs, and provides a convenient build script
+for developers at the same time.
+
 #### CodePipeline Sources
 
 In CodePipeline, *Sources* define where the source of your application lives.
@@ -756,6 +790,13 @@ class MyJenkinsStep extends pipelines.Step implements pipelines.ICodePipelineAct
     private readonly input: pipelines.FileSet,
   ) {
     super('MyJenkinsStep');
+
+    // This is necessary if your step accepts things like environment variables
+    // that may contain outputs from other steps. It doesn't matter what the
+    // structure is, as long as it contains the values that may contain outputs.
+    this.discoverReferencedOutputs({
+      env: { /* ... */ }
+    });
   }
 
   public produceAction(stage: codepipeline.IStage, options: pipelines.ProduceActionOptions): pipelines.CodePipelineActionFactoryResult {

packages/@aws-cdk/pipelines/lib/blueprint/manual-approval.ts

@@ -33,5 +33,7 @@ export class ManualApprovalStep extends Step {
     super(id);
 
     this.comment = props.comment;
+
+    this.discoverReferencedOutputs(props.comment);
   }
 }

packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts

@@ -87,7 +87,6 @@ export interface ShellStepProps {
    * @default - No primary output
    */
   readonly primaryOutputDirectory?: string;
-
 }
 
 /**
@@ -152,6 +151,11 @@ export class ShellStep extends Step {
     this.env = props.env ?? {};
     this.envFromCfnOutputs = mapValues(props.envFromCfnOutputs ?? {}, StackOutputReference.fromCfnOutput);
 
+    // 'env' is the only thing that can contain outputs
+    this.discoverReferencedOutputs({
+      env: this.env,
+    });
+
     // Inputs
     if (props.input) {
       const fileSet = props.input.primaryOutput;

packages/@aws-cdk/pipelines/lib/blueprint/step.ts

@@ -1,4 +1,5 @@
 import { Stack, Token } from '@aws-cdk/core';
+import { StepOutput } from '../helpers-internal/step-output';
 import { FileSet, IFileSetProducer } from './file-set';
 
 /**
@@ -39,7 +40,7 @@ export abstract class Step implements IFileSetProducer {
 
   private _primaryOutput?: FileSet;
 
-  private _dependencies: Step[] = [];
+  private _dependencies = new Set<Step>();
 
   constructor(
     /** Identifier for this step */
@@ -54,7 +55,10 @@ export abstract class Step implements IFileSetProducer {
    * Return the steps this step depends on, based on the FileSets it requires
    */
   public get dependencies(): Step[] {
-    return this.dependencyFileSets.map(f => f.producer).concat(this._dependencies);
+    return Array.from(new Set([
+      ...this.dependencyFileSets.map(f => f.producer),
+      ...this._dependencies,
+    ]));
   }
 
   /**
@@ -79,7 +83,7 @@ export abstract class Step implements IFileSetProducer {
    * Add a dependency on another step.
    */
   public addStepDependency(step: Step) {
-    this._dependencies.push(step);
+    this._dependencies.add(step);
   }
 
   /**
@@ -97,6 +101,21 @@ export abstract class Step implements IFileSetProducer {
   protected configurePrimaryOutput(fs: FileSet) {
     this._primaryOutput = fs;
   }
+
+  /**
+   * Crawl the given structure for references to StepOutputs and add dependencies on all steps found
+   *
+   * Should be called by subclasses based on what the user passes in as
+   * construction properties. The format of the structure passed in here does
+   * not have to correspond exactly to what gets rendered into the engine, it
+   * just needs to contain the same amount of data.
+   */
+  protected discoverReferencedOutputs(structure: any) {
+    for (const output of StepOutput.findAll(structure)) {
+      this._dependencies.add(output.step);
+      StepOutput.recordProducer(output);
+    }
+  }
 }
 
 /**
@@ -128,5 +147,4 @@ export interface StackSteps {
    * @default - no additional steps
    */
   readonly post?: Step[];
-
 }

packages/@aws-cdk/pipelines/lib/codepipeline/codebuild-step.ts

@@ -1,8 +1,10 @@
-import { Duration } from '@aws-cdk/core';
 import * as codebuild from '@aws-cdk/aws-codebuild';
 import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
+import { Duration } from '@aws-cdk/core';
 import { ShellStep, ShellStepProps } from '../blueprint';
+import { mergeBuildSpecs } from './private/buildspecs';
+import { makeCodePipelineOutput } from './private/outputs';
 
 /**
  * Construction props for a CodeBuildStep
@@ -96,6 +98,17 @@ export interface CodeBuildStepProps extends ShellStepProps {
 
 /**
  * Run a script as a CodeBuild Project
+ *
+ * The BuildSpec must be available inline--it cannot reference a file
+ * on disk. If your current build instructions are in a file like
+ * `buildspec.yml` in your repository, extract them to a script
+ * (say, `build.sh`) and invoke that script as part of the build:
+ *
+ * ```ts
+ * new pipelines.CodeBuildStep('Synth', {
+ *   commands: ['./build.sh'],
+ * });
+ * ```
  */
 export class CodeBuildStep extends ShellStep {
   /**
@@ -105,13 +118,6 @@ export class CodeBuildStep extends ShellStep {
    */
   public readonly projectName?: string;
 
-  /**
-   * Additional configuration that can only be configured via BuildSpec
-   *
-   * @default - No value specified at construction time, use defaults
-   */
-  public readonly partialBuildSpec?: codebuild.BuildSpec;
-
   /**
    * The VPC where to execute the SimpleSynth.
    *
@@ -164,13 +170,16 @@ export class CodeBuildStep extends ShellStep {
   readonly timeout?: Duration;
 
   private _project?: codebuild.IProject;
+  private _partialBuildSpec?: codebuild.BuildSpec;
+  private readonly exportedVariables = new Set<string>();
+  private exportedVarsRendered = false;
 
   constructor(id: string, props: CodeBuildStepProps) {
     super(id, props);
 
     this.projectName = props.projectName;
     this.buildEnvironment = props.buildEnvironment;
-    this.partialBuildSpec = props.partialBuildSpec;
+    this._partialBuildSpec = props.partialBuildSpec;
     this.vpc = props.vpc;
     this.subnetSelection = props.subnetSelection;
     this.role = props.role;
@@ -198,6 +207,44 @@ export class CodeBuildStep extends ShellStep {
     return this.project.grantPrincipal;
   }
 
+  /**
+   * Additional configuration that can only be configured via BuildSpec
+   *
+   * Contains exported variables
+   *
+   * @default - Contains the exported variables
+   */
+  public get partialBuildSpec(): codebuild.BuildSpec | undefined {
+    this.exportedVarsRendered = true;
+
+    const varsBuildSpec = this.exportedVariables.size > 0 ? codebuild.BuildSpec.fromObject({
+      version: '0.2',
+      env: {
+        'exported-variables': Array.from(this.exportedVariables),
+      },
+    }) : undefined;
+
+    return mergeBuildSpecs(varsBuildSpec, this._partialBuildSpec);
+  }
+
+  /**
+   * Reference a CodePipeline variable defined by the CodeBuildStep.
+   *
+   * The variable must be set in the shell of the CodeBuild step when
+   * it finishes its `post_build` phase.
+   *
+   * @param variableName the name of the variable for reference.
+   */
+  public exportedVariable(variableName: string): string {
+    if (this.exportedVarsRendered && !this.exportedVariables.has(variableName)) {
+      throw new Error('exportVariable(): Pipeline has already been produced, cannot call this function anymore');
+    }
+
+    this.exportedVariables.add(variableName);
+
+    return makeCodePipelineOutput(this, variableName);
+  }
+
   /**
    * Set the internal project value
    *

packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-action-factory.ts

@@ -23,6 +23,15 @@ export interface ProduceActionOptions {
    */
   readonly runOrder: number;
 
+  /**
+   * If this step is producing outputs, the variables namespace assigned to it
+   *
+   * Pass this on to the Action you are creating.
+   *
+   * @default - Step doesn't produce any outputs
+   */
+  readonly variablesNamespace?: string;
+
   /**
    * Helper object to translate FileSets to CodePipeline Artifacts
    */
@@ -87,6 +96,8 @@ export interface ICodePipelineActionFactory {
 export interface CodePipelineActionFactoryResult {
   /**
    * How many RunOrders were consumed
+   *
+   * If you add 1 action, return the value 1 here.
    */
   readonly runOrdersConsumed: number;