New command that allows for a user to run mobile tests that are defined as yaml.

src/apptesting/parseTestFiles.ts

@@ -5,23 +5,27 @@
 import { readFileFromDirectory, wrappedSafeLoad } from "../utils";
 import { FirebaseError, getErrMsg, getError } from "../error";
 
 function createFilter(pattern?: string) {
   const regex = pattern ? new RegExp(pattern) : undefined;
   return (s: string) => !regex || regex.test(s);
 }
 
 export async function parseTestFiles(
   dir: string,
-  targetUri: string,
+  targetUri?: string,
   filePattern?: string,
   namePattern?: string,
 ): Promise<TestCaseInvocation[]> {
-  try {
-    new URL(targetUri);
-  } catch (ex) {
-    const errMsg = "Invalid URL" + (targetUri.startsWith("http") ? "" : " (must include protocol)");
-    throw new FirebaseError(errMsg, { original: getError(ex) });
+  if (targetUri) {
+    try {
+      new URL(targetUri);
+    } catch (ex) {
+      const errMsg =
+        "Invalid URL" + (targetUri.startsWith("http") ? "" : " (must include protocol)");
+      throw new FirebaseError(errMsg, { original: getError(ex) });
+    }
   }
+
   const fileFilterFn = createFilter(filePattern);
   const nameFilterFn = createFilter(namePattern);
 
@@ -35,16 +39,16 @@
       } else if (fileFilterFn(path) && fileExistsSync(path)) {
         try {
           const file = await readFileFromDirectory(testDir, item);
           const parsedFile = wrappedSafeLoad(file.source);
           const tests = parsedFile.tests;
           const defaultConfig = parsedFile.defaultConfig;
           if (!tests || !tests.length) {
             logger.info(`No tests found in ${path}. Ignoring.`);
             continue;
           }
           for (const rawTestDef of parsedFile.tests) {
             if (!nameFilterFn(rawTestDef.testName)) continue;
-            const testDef = toTestDef(rawTestDef, targetUri, defaultConfig);
+            const testDef = toTestDef(rawTestDef, defaultConfig, targetUri);
             results.push(testDef);
           }
         } catch (ex) {
@@ -61,7 +65,7 @@
   return parseTestFilesRecursive(dir);
 }
 
-function toTestDef(testDef: any, targetUri: string, defaultConfig: any): TestCaseInvocation {
+function toTestDef(testDef: any, defaultConfig: any, targetUri?: string): TestCaseInvocation {
   const steps = testDef.steps ?? [];
   const route = testDef.testConfig?.route ?? defaultConfig?.route ?? "";
   const browsers: Browser[] = testDef.testConfig?.browsers ??

src/apptesting/types.ts

@@ -57,7 +57,7 @@ export interface TestExecutionResult {
 }
 
 export interface TestCase {
-  startUri: string;
+  startUri?: string;
   displayName: string;
   instructions: Instructions;
 }

src/commands/apptesting-mobile-execute.ts

@@ -0,0 +1,102 @@
+import { requireAuth } from "../requireAuth";
+import { Command } from "../command";
+import { logger } from "../logger";
+import * as clc from "colorette";
+import { parseTestFiles } from "../apptesting/parseTestFiles";
+import * as ora from "ora";
+import { TestCaseInvocation } from "../apptesting/types";
+import { FirebaseError, getError } from "../error";
+import { marked } from "marked";
+import { AppDistributionClient } from "../appdistribution/client";
+import { Distribution, upload } from "../appdistribution/distribution";
+import { AIInstruction, ReleaseTest } from "../appdistribution/types";
+import { getAppName } from "../appdistribution/options-parser-util";
+
+// TODO rothbutter add ability to specify devices
+const defaultDevices = [
+  {
+    model: "MediumPhone.arm",
+    version: "30",
+    locale: "en_US",
+    orientation: "portrait",
+  },
+];
+
+export const command = new Command("apptesting:mobile-execute <target>")
+  .description("Run mobile automated tests written in natural language driven by AI")
+  .option(
+    "--app <app_id>",
+    "The app id of your Firebase web app. Optional if the project contains exactly one web app.",
+  )
+  .option(
+    "--test-file-pattern <pattern>",
+    "Test file pattern. Only tests contained in files that match this pattern will be executed.",
+  )
+  .option(
+    "--test-name-pattern <pattern>",
+    "Test name pattern. Only tests with names that match this pattern will be executed.",
+  )
+  .option("--test-dir <test_dir>", "Directory where tests can be found.")
+  .before(requireAuth)
+  .action(async (target: string, options: any) => {
+    const appName = getAppName(options);
+
+    const testDir = options.testDir || "tests";
+    const tests = await parseTestFiles(
+      testDir,
+      undefined,
+      options.testFilePattern,
+      options.testNamePattern,
+    );
+
+    if (!tests.length) {
+      throw new FirebaseError("No tests found");
+    }
+
+    const invokeSpinner = ora("Requesting test execution");
+
+    let testInvocations;
+    let releaseId;
+    try {
+      const client = new AppDistributionClient();
+      releaseId = await upload(client, appName, new Distribution(target));
+
+      invokeSpinner.start();
+      testInvocations = await invokeMataTests(client, releaseId, tests);
+      invokeSpinner.text = "Test execution requested";
+      invokeSpinner.succeed();
+    } catch (ex) {
+      invokeSpinner.fail("Failed to request test execution");
+      throw ex;
+    }
+
+    logger.info(
+      clc.bold(`\n${clc.white("===")} Running ${pluralizeTests(testInvocations.length)}`),
+    );
+    logger.info(await marked(`View progress and results in the [Firebase Console]`));
+  });
+
+function pluralizeTests(numTests: number) {
+  return `${numTests} test${numTests === 1 ? "" : "s"}`;
+}
+
+async function invokeMataTests(
+  client: AppDistributionClient,
+  releaseName: string,
+  testDefs: TestCaseInvocation[],
+) {
+  try {
+    const testInvocations: ReleaseTest[] = [];
+    for (const testDef of testDefs) {
+      const aiInstruction: AIInstruction = {
+        steps: testDef.testCase.instructions.steps,
+      };
+      testInvocations.push(
+        await client.createReleaseTest(releaseName, defaultDevices, aiInstruction),
+      );
+    }
+    return testInvocations;
+  } catch (err: unknown) {
+    throw new FirebaseError("Test invocation failed", { original: getError(err) });
+  }
+}

src/commands/index.ts

@@ -265,6 +265,13 @@ export function load(client: CLIClient): CLIClient {
     client.apptesting = {};
     client.apptesting.execute = loadCommand("apptesting-execute");
   }
+  if (experiments.isEnabled("mata")) {
+    if (!client.apptesting) {
+      client.apptesting = {};
+    }
+    client.apptesting.mobile = {};
+    client.apptesting.mobile.execute = loadCommand("apptesting-mobile-execute");
+  }
 
   const t1 = process.hrtime.bigint();
   const diffMS = (t1 - t0) / BigInt(1e6);

src/experiments.ts

@@ -165,6 +165,10 @@ export const ALL_EXPERIMENTS = experiments({
     shortDescription: "Adds experimental App Testing feature",
     public: true,
   },
+  mata: {
+    shortDescription: "Adds experimental Mobile App Testing Agent (MATA) feature",
+    public: true,
+  },
   fdcwebhooks: {
     shortDescription: "Enable Firebase Data Connect webhooks feature.",
     default: false,