fix(core): account for { Ref } incompatibility between schema and CFN

packages/@aws-cdk/mixins-preview/package.json

@@ -658,7 +658,7 @@
     "@aws-cdk/integ-runner": "^2.192.2",
     "@aws-cdk/integ-tests-alpha": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",
-    "@aws-cdk/service-spec-types": "^0.0.199",
+    "@aws-cdk/service-spec-types": "^0.0.201",
     "@aws-cdk/spec2cdk": "0.0.0",
     "@cdklabs/tskb": "^0.0.4",
     "@cdklabs/typewriter": "^0.0.14",

packages/aws-cdk-lib/aws-appmesh/lib/virtual-gateway.ts

@@ -112,7 +112,6 @@ abstract class VirtualGatewayBase extends cdk.Resource implements IVirtualGatewa
   public get virtualGatewayRef(): VirtualGatewayReference {
     return {
       virtualGatewayArn: this.virtualGatewayArn,
-      virtualGatewayId: this.virtualGatewayName,
     };
   }
 

packages/aws-cdk-lib/aws-appmesh/lib/virtual-node.ts

@@ -125,7 +125,6 @@ abstract class VirtualNodeBase extends cdk.Resource implements IVirtualNode {
   public get virtualNodeRef(): VirtualNodeReference {
     return {
       virtualNodeArn: this.virtualNodeArn,
-      virtualNodeId: this.virtualNodeName,
     };
   }
 

packages/aws-cdk-lib/aws-appmesh/test/virtual-gateway.test.ts

@@ -977,12 +977,7 @@ describe('virtual gateway', () => {
           {
             Action: 'appmesh:StreamAggregatedResources',
             Effect: 'Allow',
-            Resource: {
-              'Fn::GetAtt': [
-                'testGateway',
-                'Arn',
-              ],
-            },
+            Resource: { Ref: 'testGateway' },
           },
         ],
       },

packages/aws-cdk-lib/aws-elasticsearch/lib/domain.ts

@@ -961,7 +961,7 @@ abstract class DomainBase extends cdk.Resource implements IDomain {
 
   public get domainRef(): DomainReference {
     return {
-      domainId: this.domainName,
+      domainName: this.domainName,
       domainArn: this.domainArn,
     };
   }

packages/aws-cdk-lib/package.json

@@ -131,7 +131,7 @@
   },
   "devDependencies": {
     "@aws-cdk/lambda-layer-kubectl-v31": "^2.1.0",
-    "@aws-cdk/aws-service-spec": "^0.1.133",
+    "@aws-cdk/aws-service-spec": "^0.1.135",
     "@aws-cdk/cdk-build-tools": "0.0.0",
     "@aws-cdk/custom-resource-handlers": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",

tools/@aws-cdk/spec2cdk/lib/cdk/arn.ts

@@ -7,19 +7,22 @@ import { Resource } from '@aws-cdk/service-spec-types';
  * included in the primary identifier.
  */
 export function findNonIdentifierArnProperty(resource: Resource) {
-  return findArnProperty(resource, (name) => !resource.primaryIdentifier?.includes(name));
+  const refIdentifier = resource.cfnRefIdentifier ?? resource.primaryIdentifier;
+  return findArnProperty(resource, (name) => !refIdentifier?.includes(name));
 }
 
 export function findArnProperty(resource: Resource, filter: (name: string) => boolean = () => true): string | undefined {
   const prefixes = ['', resource.name];
   const suffixes = ['Arn', 'ARN'];
   const primaryIdentifierSuffixes = ['Id', 'ID'];
 
+  const refIdentifier = resource.cfnRefIdentifier ?? resource.primaryIdentifier;
+
   // if the primary identifier uses a prefix that is different than the resource name, we add that to the list
-  if (resource.primaryIdentifier?.length === 1) {
+  if (refIdentifier?.length === 1) {
     for (const suffix of primaryIdentifierSuffixes) {
-      if (resource.primaryIdentifier[0].endsWith(suffix)) {
-        const prefix = resource.primaryIdentifier[0].slice(0, -suffix.length);
+      if (refIdentifier[0].endsWith(suffix)) {
+        const prefix = refIdentifier[0].slice(0, -suffix.length);
         if (prefix && !prefixes.includes(prefix)) {
           prefixes.push(prefix);
         }

tools/@aws-cdk/spec2cdk/lib/cdk/reference-props.ts

@@ -1,6 +1,6 @@
 import { Resource } from '@aws-cdk/service-spec-types';
 import { $this, expr, Expression, PropertySpec, Type } from '@cdklabs/typewriter';
-import { attributePropertyName, referencePropertyName } from '../naming';
+import { attributePropertyName, propertyNameFromCloudFormation, referencePropertyName } from '../naming';
 import { extractResourceVariablesFromArnFormat, findArnProperty, findNonIdentifierArnProperty } from './arn';
 import { CDK_CORE } from './cdk';
 
@@ -9,6 +9,37 @@ export interface ReferenceProp {
   readonly cfnValue: Expression;
 }
 
+/**
+ * Calculations for to the resource reference interface.
+ *
+ * This  class is slightly complicated because it needs to account for the differences between CloudFormation
+ * and CC-API identifiers.
+ *
+ * - In principle, the CC-API identifier is leading: it uniquely identifies a resource inside an environment.
+ * - For backwards compatibility reasons, the CFN identifier (the value returned by `{ Ref }`) isn't necessarily
+ *   always the same. If it isn't the same, there can be two reasons for them to diverge:
+ *
+ *    - SPECIFICITY: the CC-API identifier is more specific than the CFN
+ *      identifier. For example, for `ApiGateway::Stage`, the unique identifier is
+ *      `[ApiId, StageName]` but the value that `{ Ref }` returns is just
+ *      `StageName`.
+ *
+ *      This distinction happens for subresources, and in those cases the
+ *      primary resource will be a required input property. For maximum
+ *      flexibility we generate the interface according to the CC-API
+ *      identifier, and get values from the CFN identifier, attributes and input
+ *      properties as necessary.
+ *
+ *    - ALIASING: the CC-API uses a different form of identifying the resource than the
+ *      CFN identifier. For example, for `Batch::JobDefinition` the spec says the primary
+ *      identifier is the `Name` but the actual value that `{ Ref }` returns is the
+ *      the `Arn`. We will just use the CFN value as leading.
+ *
+ * We will identify the difference between these 2 cases by the length of the primary
+ * identifier: equal length = aliasing, different length = specificity.
+ *
+ * If available, we also add an ARN field into the reference interface.
+ */
 export class ResourceReference {
   public readonly resource: Resource;
   public readonly arnPropertyName?: string;
@@ -44,36 +75,20 @@ export class ResourceReference {
   }
 
   private collectReferencesProps() {
-    // Primary identifier. We assume all parts are strings.
-    const primaryIdentifier = this.resource.primaryIdentifier ?? [];
-    if (primaryIdentifier.length === 1) {
-      const name = referencePropertyName(primaryIdentifier[0], this.resource.name);
+    // Reference fields
+    for (const cfnName of this.referenceFields) {
+      const name = referencePropertyName(cfnName, this.resource.name);
       this._referenceProps.setIfAbsent(name, {
         declaration: {
           name,
           type: Type.STRING,
           immutable: true,
           docs: {
-            summary: `The ${primaryIdentifier[0]} of the ${this.resource.name} resource.`,
+            summary: `The ${cfnName} of the ${this.resource.name} resource.`,
           },
         },
-        cfnValue: $this.ref,
+        cfnValue: this.getStringValue(cfnName),
       });
-    } else if (primaryIdentifier.length > 1) {
-      for (const [i, cfnName] of primaryIdentifier.entries()) {
-        const name = referencePropertyName(cfnName, this.resource.name);
-        this._referenceProps.setIfAbsent(name, {
-          declaration: {
-            name,
-            type: Type.STRING,
-            immutable: true,
-            docs: {
-              summary: `The ${cfnName} of the ${this.resource.name} resource.`,
-            },
-          },
-          cfnValue: splitSelect('|', i, $this.ref),
-        });
-      }
     }
 
     // Arn identifier
@@ -137,6 +152,71 @@ export class ResourceReference {
     // we don't have a matching value
     return undefined;
   }
+
+  /**
+   * The actual reference fields
+   *
+   * The CFN values if present and the same length as the CC-API values, otherwise the CC-API values.
+   *
+   * For a CC-API identifier we filter out optional properties, such as for `ECS::Cluster`: the real
+   * unique identifier includes `Cluster` but that is an optional property because the Service will fall
+   * back to some implicit default Cluster that we can never replicate.
+   */
+  public get referenceFields(): string[] {
+    if (this.resource.cfnRefIdentifier && this.resource.cfnRefIdentifier.length === this.resource.primaryIdentifier?.length) {
+      return this.resource.cfnRefIdentifier;
+    }
+
+    // Filter out properties we can't find a value for (will only be optional properties)
+    return (this.resource.primaryIdentifier ?? [])
+      .filter(p => this.tryGetStringValue(p) !== undefined);
+  }
+
+  /**
+   * What `{ Ref }` returns in CloudFormation
+   */
+  public get cfnRefComponents(): string[] {
+    return this.resource.cfnRefIdentifier ?? this.resource.primaryIdentifier ?? [];
+  }
+
+  /**
+   * Return an expression to return the given value from the { Ref } or any of the attributes or properties
+   */
+  private tryGetStringValue(name: string): Expression | undefined {
+    for (const [i, field] of this.cfnRefComponents.entries()) {
+      if (field === name) {
+        // Return entire field or Split expression, depending on whether we need to split at all
+        return this.cfnRefComponents.length > 1 ? splitSelect('|', i, $this.ref) : $this.ref;
+      }
+    }
+
+    // Is it an attr?
+    if (this.resource.attributes[name]) {
+      return $this[attributePropertyName(name)];
+    }
+
+    // A required prop?
+    if (this.resource.properties[name]?.required) {
+      return $this[propertyNameFromCloudFormation(name)];
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Return a value, failing if it doesn't exist.
+   */
+  private getStringValue(name: string): Expression {
+    const ret = this.tryGetStringValue(name);
+    if (ret) {
+      return ret;
+    }
+
+    const attributeNames = Object.keys(this.resource.attributes);
+    const requiredPropertyNames = Object.entries(this.resource.properties).filter(([_, p]) => p.required).map(([n, _]) => n);
+
+    throw new Error(`Cannot find reference interface value name ${name} for resource ${this.resource.cloudFormationType} (Ref components: ${this.cfnRefComponents}, attributes: ${attributeNames}, requiredProps: ${requiredPropertyNames})`);
+  }
 }
 
 class FirstOccurrenceMap<K, V> extends Map<K, V> {

tools/@aws-cdk/spec2cdk/package.json

@@ -31,9 +31,9 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@aws-cdk/aws-service-spec": "^0.1.133",
+    "@aws-cdk/aws-service-spec": "^0.1.135",
     "@aws-cdk/service-spec-importers": "^0.0.101",
-    "@aws-cdk/service-spec-types": "^0.0.199",
+    "@aws-cdk/service-spec-types": "^0.0.201",
     "@cdklabs/tskb": "^0.0.4",
     "@cdklabs/typewriter": "^0.0.14",
     "camelcase": "^6",
@@ -46,6 +46,7 @@
     "@aws-cdk/pkglint": "0.0.0",
     "@types/jest": "^29.5.14",
     "@types/node": "^18",
+    "diff": "^8.0.2",
     "jest": "^29.7.0"
   },
   "keywords": [

tools/@aws-cdk/spec2cdk/test/__snapshots__/resources.test.ts.snap

@@ -189,7 +189,7 @@ function CfnResourcePropsFromCloudFormation(properties: any): cfn_parse.FromClou
 export type { IResourceRef, ResourceReference };"
 `;
 
-exports[`resource interface when primaryIdentifier is a property 1`] = `
+exports[`resource interface when cfnRefIdentifier is a property 1`] = `
 "/* eslint-disable prettier/prettier, @stylistic/max-len */
 import * as cdk from "aws-cdk-lib/core";
 import * as constructs from "constructs";

tools/@aws-cdk/spec2cdk/test/expect-to-contain-code.ts

@@ -0,0 +1,54 @@
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { ChangeObject, diffLines } from 'diff';
+
+expect.extend({
+  toContainCode(actual: string, expected: string) {
+    // actual = actual.split('\n').map(x => x.trim()).join('\n');
+    // subset = subset.split('\n').map(x => x.trim()).join('\n');
+    // console.log(actual);
+    const ds = diffLines(actual, expected, {
+      ignoreWhitespace: true,
+    });
+
+    // We expect to see only 'removed' lines at the edges (lines in the "actual"
+    // we can't find in the "expected"). Then, what remains should match exactly.
+    while (ds.length > 0 && !ds[0].added && ds[0].removed) {
+      ds.splice(0, 1);
+    }
+    while (ds.length > 0 && !ds[ds.length - 1].added && ds[ds.length - 1].removed) {
+      ds.splice(ds.length - 1, 1);
+    }
+
+    const ok = ds.every(d => !d.added && !d.removed);
+
+    return {
+      pass: ok,
+      message: () => renderDiff(ds),
+    };
+  },
+});
+
+declare global {
+  namespace jest {
+    // Optionally, also as an Asymmetric Matcher
+    interface Matchers<R> {
+      toContainCode(expected: string): CustomMatcherResult;
+    }
+  }
+}
+
+function renderDiff(ds: ChangeObject<string>[]) {
+  const ret = new Array<string>();
+  for (const d of ds) {
+    // Always ends with a `\n` we're not interested in
+    let lines = d.value.slice(0, -1).split('\n');
+    if (d.added) {
+      lines = lines.map(x => `+${x}`);
+    }
+    if (d.removed) {
+      lines = lines.map(x => `-${x}`);
+    }
+    ret.push(...lines);
+  }
+  return ret.join('\n');
+}