refactor(cli): create ProxyAgent in the CLI in preparation of proxy-agent removal from toolkit-lib (#532)

Updates the `CdkToolkit` and `Notices` APIs to take a Node https.Agent
instead of options. This gives more flexibility to integrators while
retaining proxy support. We can make the changes to `Notices` since it
is not a public API.

The CLI changed need to be split out from the toolkit-lib changes, so
the latter can be safely marked as breaking without touching CLI files.
The respective `toolkit-lib` changes are in #533

Relates to #398 

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

Author: mrgrain

packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/awscli-compatible.ts

@@ -1,3 +1,4 @@
+import type { Agent } from 'node:https';
 import { format } from 'node:util';
 import type { SDKv3CompatibleCredentialProvider } from '@aws-cdk/cli-plugin-contract';
 import { createCredentialChain, fromEnv, fromIni, fromNodeProviderChain } from '@aws-sdk/credential-providers';
@@ -270,13 +271,16 @@ export interface CredentialChainOptions {
   readonly logger?: ISdkLogger;
 }
 
-export async function makeRequestHandler(ioHelper: IoHelper, options: SdkHttpOptions = {}): Promise<NodeHttpHandlerOptions> {
-  const agent = await new ProxyAgentProvider(ioHelper).create(options);
-
+export function sdkRequestHandler(agent?: Agent): NodeHttpHandlerOptions {
   return {
     connectionTimeout: DEFAULT_CONNECTION_TIMEOUT,
     requestTimeout: DEFAULT_TIMEOUT,
     httpsAgent: agent,
     httpAgent: agent,
   };
 }
+
+export async function makeRequestHandler(ioHelper: IoHelper, options: SdkHttpOptions = {}): Promise<NodeHttpHandlerOptions> {
+  const agent = await new ProxyAgentProvider(ioHelper).create(options);
+  return sdkRequestHandler(agent);
+}

packages/@aws-cdk/toolkit-lib/lib/api/notices/notices.ts

@@ -1,6 +1,6 @@
+import type { Agent } from 'https';
 import * as path from 'path';
 import { cdkCacheDir } from '../../util';
-import type { SdkHttpOptions } from '../aws-auth';
 import type { Context } from '../context';
 import type { IIoHost } from '../io';
 import { CachedDataSource } from './cached-data-source';
@@ -13,6 +13,20 @@ import { IO, asIoHelper } from '../io/private';
 
 const CACHE_FILE_PATH = path.join(cdkCacheDir(), 'notices.json');
 
+/**
+ * Options for the HTTPS requests made by Notices
+ */
+export interface NoticesHttpOptions {
+  /**
+   * The agent responsible for making the network requests.
+   *
+   * Use this so set up a proxy connection.
+   *
+   * @default - uses the shared global node agent
+   */
+  readonly agent?: Agent;
+}
+
 export interface NoticesProps {
   /**
    * CDK context
@@ -27,9 +41,9 @@ export interface NoticesProps {
   readonly output?: string;
 
   /**
-   * Options for the HTTP request
+   * Options for the HTTPS requests made by Notices
    */
-  readonly httpOptions?: SdkHttpOptions;
+  readonly httpOptions?: NoticesHttpOptions;
 
   /**
    * Where messages are going to be sent
@@ -100,7 +114,7 @@ export class Notices {
   private readonly context: Context;
   private readonly output: string;
   private readonly acknowledgedIssueNumbers: Set<Number>;
-  private readonly httpOptions: SdkHttpOptions;
+  private readonly httpOptions: NoticesHttpOptions;
   private readonly ioHelper: IoHelper;
   private readonly cliVersion: string;
 
@@ -148,7 +162,7 @@ export class Notices {
   }
 
   /**
-   * Filter the data sourece for relevant notices
+   * Filter the data source for relevant notices
    */
   public filter(options: NoticesDisplayOptions = {}): Promise<FilteredNotice[]> {
     return new NoticesFilter(this.ioHelper).filter({

packages/@aws-cdk/toolkit-lib/lib/api/notices/web-data-source.ts

@@ -1,25 +1,53 @@
-import type { ClientRequest } from 'http';
-import type { RequestOptions } from 'https';
+import type { ClientRequest } from 'node:http';
+import type { RequestOptions } from 'node:https';
 import * as https from 'node:https';
-import type { SdkHttpOptions } from '../aws-auth';
 import type { Notice, NoticeDataSource } from './types';
 import { ToolkitError } from '../../toolkit/toolkit-error';
 import { formatErrorMessage, humanHttpStatusError, humanNetworkError } from '../../util';
-import { ProxyAgentProvider } from '../aws-auth/private';
 import type { IoHelper } from '../io/private';
 
+/**
+ * A data source that fetches notices from the CDK notices data source
+ */
+export class WebsiteNoticeDataSourceProps {
+  /**
+   * The URL to load notices from.
+   *
+   * Note this must be a valid JSON document in the CDK notices data schema.
+   *
+   * @see https://github.com/cdklabs/aws-cdk-notices
+   *
+   * @default - official CDK notices
+   */
+  readonly url?: string | URL;
+  /**
+   * The agent responsible for making the network requests.
+   *
+   * Use this so set up a proxy connection.
+   *
+   * @default - uses the shared global node agent
+   */
+  readonly agent?: https.Agent;
+}
+
 export class WebsiteNoticeDataSource implements NoticeDataSource {
-  private readonly options: SdkHttpOptions;
+  /**
+   * The URL notices are loaded from.
+   */
+  public readonly url: any;
+
+  private readonly agent?: https.Agent;
 
-  constructor(private readonly ioHelper: IoHelper, options: SdkHttpOptions = {}) {
-    this.options = options;
+  constructor(private readonly ioHelper: IoHelper, props: WebsiteNoticeDataSourceProps = {}) {
+    this.agent = props.agent;
+    this.url = props.url ?? 'https://cli.cdk.dev-tools.aws.dev/notices.json';
   }
 
   async fetch(): Promise<Notice[]> {
     const timeout = 3000;
 
     const options: RequestOptions = {
-      agent: await new ProxyAgentProvider(this.ioHelper).create(this.options),
+      agent: this.agent,
     };
 
     const notices = await new Promise<Notice[]>((resolve, reject) => {
@@ -34,7 +62,7 @@ export class WebsiteNoticeDataSource implements NoticeDataSource {
       timer.unref();
 
       try {
-        req = https.get('https://cli.cdk.dev-tools.aws.dev/notices.json',
+        req = https.get(this.url,
           options,
           res => {
             if (res.statusCode === 200) {

packages/aws-cdk/lib/cli/cdk-toolkit.ts

@@ -136,9 +136,15 @@ export enum AssetBuildTime {
   JUST_IN_TIME = 'just-in-time',
 }
 
+/**
+ * Custom implementation of the public Toolkit to integrate with the legacy CdkToolkit
+ *
+ * This overwrites how an sdkProvider is acquired
+ * in favor of the one provided directly to CdkToolkit.
+ */
 class InternalToolkit extends Toolkit {
   private readonly _sdkProvider: SdkProvider;
-  public constructor(sdkProvider: SdkProvider, options: ToolkitOptions) {
+  public constructor(sdkProvider: SdkProvider, options: Omit<ToolkitOptions, 'sdkConfig'>) {
     super(options);
     this._sdkProvider = sdkProvider;
   }
@@ -172,10 +178,8 @@ export class CdkToolkit {
       color: true,
       emojis: true,
       ioHost: this.ioHost,
-      sdkConfig: {},
       toolkitStackName: this.toolkitStackName,
     });
-    this.toolkit; // aritifical use of this.toolkit to satisfy TS, we want to prepare usage of the new toolkit without using it just yet
   }
 
   public async metadata(stackName: string, json: boolean) {

packages/aws-cdk/lib/cli/cli.ts

@@ -16,7 +16,7 @@ import * as version from './version';
 import { asIoHelper } from '../../lib/api-private';
 import type { IReadLock } from '../api';
 import { ToolkitInfo, Notices } from '../api';
-import { SdkProvider, IoHostSdkLogger, setSdkTracing, makeRequestHandler } from '../api/aws-auth';
+import { SdkProvider, IoHostSdkLogger, setSdkTracing, sdkRequestHandler } from '../api/aws-auth';
 import type { BootstrapSource } from '../api/bootstrap';
 import { Bootstrapper } from '../api/bootstrap';
 import { Deployments } from '../api/deployments';
@@ -29,6 +29,7 @@ import { cliInit, printAvailableTemplates } from '../commands/init';
 import { getMigrateScanType } from '../commands/migrate';
 import { execProgram, CloudExecutable } from '../cxapp';
 import type { StackSelector, Synthesizer } from '../cxapp';
+import { ProxyAgentProvider } from './proxy-agent';
 
 if (!process.stdout.isTTY) {
   // Disable chalk color highlighting
@@ -89,17 +90,20 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
 
   const ioHelper = asIoHelper(ioHost, ioHost.currentAction as any);
 
+  // Always create and use ProxyAgent to support configuration via env vars
+  const proxyAgent = await new ProxyAgentProvider(ioHelper).create({
+    proxyAddress: configuration.settings.get(['proxy']),
+    caBundlePath: configuration.settings.get(['caBundlePath']),
+  });
+
   const shouldDisplayNotices = configuration.settings.get(['notices']);
   // Notices either go to stderr, or nowhere
   ioHost.noticesDestination = shouldDisplayNotices ? 'stderr' : 'drop';
   const notices = Notices.create({
     ioHost,
     context: configuration.context,
     output: configuration.settings.get(['outdir']),
-    httpOptions: {
-      proxyAddress: configuration.settings.get(['proxy']),
-      caBundlePath: configuration.settings.get(['caBundlePath']),
-    },
+    httpOptions: { agent: proxyAgent },
     cliVersion: version.versionNumber(),
   });
   const refreshNotices = (async () => {
@@ -116,10 +120,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
   const sdkProvider = await SdkProvider.withAwsCliCompatibleDefaults({
     ioHelper,
     profile: configuration.settings.get(['profile']),
-    requestHandler: await makeRequestHandler(ioHelper, {
-      proxyAddress: argv.proxy,
-      caBundlePath: argv['ca-bundle-path'],
-    }),
+    requestHandler: sdkRequestHandler(proxyAgent),
     logger: new IoHostSdkLogger(asIoHelper(ioHost, ioHost.currentAction as any)),
     pluginHost: GLOBAL_PLUGIN_HOST,
   });

packages/aws-cdk/lib/cli/proxy-agent.ts

@@ -0,0 +1,73 @@
+import * as fs from 'fs-extra';
+import { ProxyAgent } from 'proxy-agent';
+import type { IoHelper } from '../api-private';
+
+/**
+ * Options for proxy-agent SDKs
+ */
+interface ProxyAgentOptions {
+  /**
+   * Proxy address to use
+   *
+   * @default No proxy
+   */
+  readonly proxyAddress?: string;
+
+  /**
+   * A path to a certificate bundle that contains a cert to be trusted.
+   *
+   * @default No certificate bundle
+   */
+  readonly caBundlePath?: string;
+}
+
+export class ProxyAgentProvider {
+  private readonly ioHelper: IoHelper;
+
+  public constructor(ioHelper: IoHelper) {
+    this.ioHelper = ioHelper;
+  }
+
+  public async create(options: ProxyAgentOptions) {
+    // Force it to use the proxy provided through the command line.
+    // Otherwise, let the ProxyAgent auto-detect the proxy using environment variables.
+    const getProxyForUrl = options.proxyAddress != null
+      ? () => Promise.resolve(options.proxyAddress!)
+      : undefined;
+
+    return new ProxyAgent({
+      ca: await this.tryGetCACert(options.caBundlePath),
+      getProxyForUrl,
+    });
+  }
+
+  private async tryGetCACert(bundlePath?: string) {
+    const path = bundlePath || this.caBundlePathFromEnvironment();
+    if (path) {
+      await this.ioHelper.defaults.debug(`Using CA bundle path: ${path}`);
+      try {
+        if (!fs.pathExistsSync(path)) {
+          return undefined;
+        }
+        return fs.readFileSync(path, { encoding: 'utf-8' });
+      } catch (e: any) {
+        await this.ioHelper.defaults.debug(String(e));
+        return undefined;
+      }
+    }
+    return undefined;
+  }
+
+  /**
+   * Find and return a CA certificate bundle path to be passed into the SDK.
+   */
+  private caBundlePathFromEnvironment(): string | undefined {
+    if (process.env.aws_ca_bundle) {
+      return process.env.aws_ca_bundle;
+    }
+    if (process.env.AWS_CA_BUNDLE) {
+      return process.env.AWS_CA_BUNDLE;
+    }
+    return undefined;
+  }
+}