Introduce IReadableCloudAssembly

Author: rix0rrr

packages/@aws-cdk/tmp-toolkit-helpers/src/api/rwlock.ts

@@ -29,7 +29,7 @@ export class RWLock {
    *
    * No other readers or writers must exist for the given directory.
    */
-  public async acquireWrite(): Promise<IWriterLock> {
+  public async acquireWrite(): Promise<IWriteLock> {
     await this.assertNoOtherWriters();
 
     const readers = await this.currentReaders();
@@ -39,9 +39,14 @@ export class RWLock {
 
     await writeFileAtomic(this.writerFile, this.pidString);
 
+    let released = false;
     return {
       release: async () => {
-        await deleteFile(this.writerFile);
+        // Releasing needs a flag, otherwise we might delete a file that some other lock has created in the mean time.
+        if (!released) {
+          await deleteFile(this.writerFile);
+          released = true;
+        }
       },
       convertToReaderLock: async () => {
         // Acquire the read lock before releasing the write lock. Slightly less
@@ -58,7 +63,7 @@ export class RWLock {
    *
    * Will fail if there are any writers.
    */
-  public async acquireRead(): Promise<ILock> {
+  public async acquireRead(): Promise<IReadLock> {
     await this.assertNoOtherWriters();
     return this.doAcquireRead();
   }
@@ -77,12 +82,18 @@ export class RWLock {
   /**
    * Do the actual acquiring of a read lock.
    */
-  private async doAcquireRead(): Promise<ILock> {
+  private async doAcquireRead(): Promise<IReadLock> {
     const readerFile = this.readerFile();
     await writeFileAtomic(readerFile, this.pidString);
+
+    let released = false;
     return {
       release: async () => {
-        await deleteFile(readerFile);
+        // Releasing needs a flag, otherwise we might delete a file that some other lock has created in the mean time.
+        if (!released) {
+          await deleteFile(readerFile);
+          released = true;
+        }
       },
     };
   }
@@ -151,18 +162,21 @@ export class RWLock {
 /**
  * An acquired lock
  */
-export interface ILock {
+export interface IReadLock {
+  /**
+   * Release the lock. Can be called more than once.
+   */
   release(): Promise<void>;
 }
 
 /**
  * An acquired writer lock
  */
-export interface IWriterLock extends ILock {
+export interface IWriteLock extends IReadLock {
   /**
    * Convert the writer lock to a reader lock
    */
-  convertToReaderLock(): Promise<ILock>;
+  convertToReaderLock(): Promise<IReadLock>;
 }
 
 /* c8 ignore start */ // code paths are unpredictable

packages/@aws-cdk/toolkit-lib/lib/actions/bootstrap/index.ts

@@ -25,7 +25,7 @@ export class BootstrapEnvironments {
   static fromCloudAssemblySource(cx: ICloudAssemblySource): BootstrapEnvironments {
     return new BootstrapEnvironments(async (ioHost: IIoHost) => {
       const ioHelper = asIoHelper(ioHost, 'bootstrap');
-      const assembly = await assemblyFromSource(ioHelper, cx);
+      await using assembly = await assemblyFromSource(ioHelper, cx);
       const stackCollection = await assembly.selectStacksV2(ALL_STACKS);
       return stackCollection.stackArtifacts.map(stack => stack.environment);
     });

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/cached-source.ts

@@ -0,0 +1,50 @@
+import type * as cxapi from '@aws-cdk/cx-api';
+import { BorrowedAssembly } from './private/borrowed-assembly';
+import type { ICloudAssemblySource, IReadableCloudAssembly } from './types';
+
+/**
+ * A CloudAssemblySource that is caching its result once produced.
+ *
+ * Most Toolkit interactions should use a cached source. Not caching is
+ * relevant when the source changes frequently and it is to expensive to predict
+ * if the source has changed.
+ *
+ * The `CachedCloudAssembly` is both itself a readable CloudAssembly, as well as
+ * a Cloud Assembly Source. The lifetimes of cloud assemblies produced by this
+ * source are coupled to the lifetime of the `CachedCloudAssembly`. In other
+ * words: the `dispose()` functions of those cloud assemblies don't do anything;
+ * only the `dispose()` function of the `CachedCloudAssembly` will be used.
+ *
+ * FIXME: We should consider referencing counting here, although that seems
+ * unnecessarily complicated for now. Be aware to callers that failing to dispose
+ * the result if a `produce()` call of a `CachedCloudAssembly` is considered
+ * a bug and may lead to resource leakage in the future, even though it
+ * might work today.
+ */
+export class CachedCloudAssembly implements ICloudAssemblySource, IReadableCloudAssembly {
+  private asm: IReadableCloudAssembly;
+
+  public constructor(asm: IReadableCloudAssembly) {
+    this.asm = asm;
+  }
+
+  public get cloudAssembly(): cxapi.CloudAssembly {
+    return this.asm.cloudAssembly;
+  }
+
+  public async produce(): Promise<IReadableCloudAssembly> {
+    return new BorrowedAssembly(this.asm.cloudAssembly);
+  }
+
+  public _unlock() {
+    return this.asm._unlock();
+  }
+
+  public dispose(): Promise<void> {
+    return this.asm.dispose();
+  }
+
+  public [Symbol.asyncDispose](): Promise<void> {
+    return this.dispose();
+  }
+}

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/index.ts

@@ -1,4 +1,5 @@
 export { StackSelectionStrategy, StackSelector } from '../../api/shared-public';
+export * from './cached-source';
 export * from './source-builder';
 export * from './types';
 

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/private/borrowed-assembly.ts

@@ -0,0 +1,22 @@
+import type * as cxapi from '@aws-cdk/cx-api';
+import type { IReadableCloudAssembly } from '../types';
+
+/**
+ * An implementation of `IReadableCloudAssembly` that does nothing except hold on to the CloudAssembly object
+ *
+ * It does not own a lock, and it does not clean the underlying directory.
+ */
+export class BorrowedAssembly implements IReadableCloudAssembly {
+  constructor(public readonly cloudAssembly: cxapi.CloudAssembly) {
+  }
+
+  public async _unlock(): Promise<void> {
+  }
+
+  public async dispose(): Promise<void> {
+  }
+
+  public async [Symbol.asyncDispose](): Promise<void> {
+  }
+}
+

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/private/cached-source.ts

@@ -1,11 +1,10 @@
 import type { MissingContext } from '@aws-cdk/cloud-assembly-schema';
-import type * as cxapi from '@aws-cdk/cx-api';
 import type { ToolkitServices } from '../../../toolkit/private';
 import { IO } from '../../io/private';
 import { contextproviders } from '../../shared-private';
 import { PROJECT_CONTEXT, type Context, type IoHelper } from '../../shared-private';
 import { ToolkitError } from '../../shared-public';
-import type { ICloudAssemblySource } from '../types';
+import type { ICloudAssemblySource, IReadableCloudAssembly } from '../types';
 
 export interface ContextAwareCloudAssemblyProps {
   /**
@@ -37,9 +36,22 @@ export interface ContextAwareCloudAssemblyProps {
 }
 
 /**
- * Represent the Cloud Executable and the synthesis we can do on it
+ * A CloudAssemblySource that wraps another CloudAssemblySource and runs a lookup loop on it
+ *
+ * This means that if the underlying CloudAssemblySource produces a manifest
+ * with provider queries in it, the `ContextAwareCloudAssemblySource` will
+ * perform the necessary context lookups and invoke the underlying
+ * `CloudAssemblySource` again with thew missing context information.
+ *
+ * This is only useful if the underlying `CloudAssemblySource` can respond to
+ * this new context information (it must be a CDK app source); if it is just a
+ * static directory,
+ *
+ * The context is passed between `ContextAwareCloudAssemblySource` and the wrapped
+ * cloud assembly source via a contex file on disk, so the wrapped assembly source
+ * should re-read the context file on every invocation.
  */
-export class ContextAwareCloudAssembly implements ICloudAssemblySource {
+export class ContextAwareCloudAssemblySource implements ICloudAssemblySource {
   private canLookup: boolean;
   private context: Context;
   private contextFile: string;
@@ -55,15 +67,16 @@ export class ContextAwareCloudAssembly implements ICloudAssemblySource {
   /**
    * Produce a Cloud Assembly, i.e. a set of stacks
    */
-  public async produce(): Promise<cxapi.CloudAssembly> {
+  public async produce(): Promise<IReadableCloudAssembly> {
     // We may need to run the cloud assembly source multiple times in order to satisfy all missing context
     // (When the source producer runs, it will tell us about context it wants to use
     // but it missing. We'll then look up the context and run the executable again, and
     // again, until it doesn't complain anymore or we've stopped making progress).
     let previouslyMissingKeys: Set<string> | undefined;
     while (true) {
-      const assembly = await this.source.produce();
+      const readableAsm = await this.source.produce();
 
+      const assembly = readableAsm.cloudAssembly;
       if (assembly.manifest.missing && assembly.manifest.missing.length > 0) {
         const missingKeysSet = missingContextKeys(assembly.manifest.missing);
         const missingKeys = Array.from(missingKeysSet);
@@ -100,11 +113,12 @@ export class ContextAwareCloudAssembly implements ICloudAssemblySource {
           await this.context.save(this.contextFile);
 
           // Execute again
+          // TODO: Unlock
           continue;
         }
       }
 
-      return assembly;
+      return readableAsm;
     }
   }
 }

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/private/context-aware-source.ts

@@ -1,5 +1,4 @@
 export * from './context-aware-source';
-export * from './cached-source';
 export * from './identity-source';
 export * from './stack-assembly';
 export * from './exec';

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/private/identity-source.ts

@@ -5,8 +5,8 @@ import * as cxschema from '@aws-cdk/cloud-assembly-schema';
 import * as cxapi from '@aws-cdk/cx-api';
 import * as fs from 'fs-extra';
 import { lte } from 'semver';
-import { prepareDefaultEnvironment as oldPrepare, prepareContext, spaceAvailableForContext, guessExecutable } from '../../../api/shared-private';
 import { type SdkProvider, type IoHelper, loadTree, some, Settings } from '../../../api/shared-private';
+import { prepareDefaultEnvironment as oldPrepare, prepareContext, spaceAvailableForContext, guessExecutable } from '../../../api/shared-private';
 import { splitBySize, versionNumber } from '../../../private/util';
 import type { ToolkitServices } from '../../../toolkit/private';
 import { IO } from '../../io/private';
@@ -17,6 +17,8 @@ type Env = { [key: string]: string };
 type Context = { [key: string]: any };
 
 export class ExecutionEnvironment {
+  public readonly outDirIsTemporary: boolean;
+
   private readonly ioHelper: IoHelper;
   private readonly sdkProvider: SdkProvider;
   private readonly debugFn: (msg: string) => Promise<void>;
@@ -27,6 +29,7 @@ export class ExecutionEnvironment {
     this.sdkProvider = services.sdkProvider;
     this.debugFn = (msg: string) => this.ioHelper.notify(IO.DEFAULT_ASSEMBLY_DEBUG.msg(msg));
     this._outdir = props.outdir;
+    this.outDirIsTemporary = props.outdir === undefined;
   }
 
   /**