feat: Added DQL Metadata like scannedBytes to execute_dql tool (#124)

* feat: Added DQL Metadata like scannedBytes to execute_dql tool

* Review comments

* Apply suggestions from code review

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: christian-kreuzberger-dtx

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased Changes
 
+- Added metadata output which includes scanned bytes (for cost tracking) to `execute_dql`
+
 ## 0.5.0
 
 **Highlights**:

README.md

@@ -27,19 +27,29 @@ Bring real-time observability data directly into your development workflow.
 - Get more information about a monitored entity
 - Get Ownership of an entity
 
-## Costs
+### Costs
 
-**Important:** While this local MCP server is provided for free, using it to access data in Dynatrace Grail may incur additional costs based
+**Important:** While this local MCP server is provided for free, using certain capabilities to access data in Dynatrace Grail may incur additional costs based
 on your Dynatrace consumption model. This affects `execute_dql` tool and other capabilities that **query** Dynatrace Grail storage, and costs
-depend on the volume (GB scanned/billed).
+depend on the volume (GB scanned).
 
 **Before using this MCP server extensively, please:**
 
 1. Review your current Dynatrace consumption model and pricing
 2. Understand the cost implications of the specific data you plan to query (logs, events, metrics) - see [Dynatrace Pricing and Rate Card](https://www.dynatrace.com/pricing/)
 3. Start with smaller timeframes (e.g., 12h-24h) and make use of [buckets](https://docs.dynatrace.com/docs/discover-dynatrace/platform/grail/data-model#built-in-grail-buckets) to reduce the cost impact
 
-**Note**: We will be providing a way to monitor Query Usage of the dynatrace-mcp-server in the future.
+**To understand costs that occured:**
+
+Execute the following DQL statement in a notebook to see how much bytes have been queried from Grail (Logs, Events, etc...):
+
+```
+fetch dt.system.events
+| filter event.kind == "QUERY_EXECUTION_EVENT" and contains(client.client_context, "dynatrace-mcp")
+| sort timestamp desc
+| fields timestamp, query_id, query_string, scanned_bytes, table, bucket, user.id, user.email, client.client_context
+| maketimeSeries sum(scanned_bytes), by: { user.email, user.id, table }
+```
 
 ### AI-Powered Assistance (Preview)
 

integration-tests/execute-dql.integration.test.ts

@@ -95,13 +95,20 @@ describe('Execute DQL Integration Tests', () => {
     });
 
     expect(executionResponse).toBeDefined();
-    expect(Array.isArray(executionResponse)).toBe(true);
+    expect(executionResponse?.records).toBeDefined();
+    expect(Array.isArray(executionResponse?.records)).toBe(true);
 
     // Should return an array of records (even if empty)
-    if (executionResponse && executionResponse.length > 0) {
+    if (executionResponse?.records && executionResponse.records.length > 0) {
       // Check that records have expected structure
-      expect(typeof executionResponse[0]).toBe('object');
+      expect(typeof executionResponse.records[0]).toBe('object');
     }
+
+    // Check that cost information is available
+    expect(executionResponse?.scannedBytes).toBeDefined();
+    expect(typeof executionResponse?.scannedBytes).toBe('number');
+    expect(executionResponse?.scannedRecords).toBeDefined();
+    expect(typeof executionResponse?.scannedRecords).toBe('number');
   });
 
   test('should execute metrics query', async () => {
@@ -122,11 +129,12 @@ describe('Execute DQL Integration Tests', () => {
     });
 
     expect(executionResponse).toBeDefined();
-    expect(Array.isArray(executionResponse)).toBe(true);
+    expect(executionResponse?.records).toBeDefined();
+    expect(Array.isArray(executionResponse?.records)).toBe(true);
 
     // Metrics might not always have data, so we just check structure
-    if (executionResponse && executionResponse.length > 0) {
-      expect(typeof executionResponse[0]).toBe('object');
+    if (executionResponse?.records && executionResponse.records.length > 0) {
+      expect(typeof executionResponse.records[0]).toBe('object');
     }
   });
 
@@ -148,13 +156,14 @@ describe('Execute DQL Integration Tests', () => {
     });
 
     expect(executionResponse).toBeDefined();
-    expect(Array.isArray(executionResponse)).toBe(true);
+    expect(executionResponse?.records).toBeDefined();
+    expect(Array.isArray(executionResponse?.records)).toBe(true);
 
     // Events might not always have data, so we just check structure
-    if (executionResponse && executionResponse.length > 0) {
-      expect(typeof executionResponse[0]).toBe('object');
+    if (executionResponse?.records && executionResponse.records.length > 0) {
+      expect(typeof executionResponse.records[0]).toBe('object');
       // Events should have common fields like timestamp, event.type, etc.
-      const firstEvent = executionResponse[0] as Record<string, any>;
+      const firstEvent = executionResponse.records[0] as Record<string, any>;
       expect(firstEvent).toHaveProperty('timestamp');
     }
   });

src/capabilities/execute-dql.ts

@@ -14,30 +14,70 @@ export const verifyDqlStatement = async (dtClient: HttpClient, dqlStatement: str
   return response;
 };
 
+export interface DqlExecutionResult {
+  records: QueryResult['records'];
+  metadata: QueryResult['metadata'];
+  /** Number of Bytes scanned = Bytes Billed */
+  scannedBytes?: number;
+  scannedRecords?: number;
+  executionTimeMilliseconds?: number;
+  queryId?: string;
+  sampled?: boolean;
+}
+
+/**
+ * Helper function to create a DQL execution result and log metadata information.
+ * @param queryResult - The query result from Dynatrace API
+ * @param logPrefix - Prefix for the log message (e.g., "DQL Execution Metadata" or "DQL Execution Metadata (Polled)")
+ * @returns DqlExecutionResult with extracted metadata
+ */
+const createResultAndLog = (queryResult: QueryResult, logPrefix: string): DqlExecutionResult => {
+  const result: DqlExecutionResult = {
+    records: queryResult.records,
+    metadata: queryResult.metadata,
+    scannedBytes: queryResult.metadata?.grail?.scannedBytes,
+    scannedRecords: queryResult.metadata?.grail?.scannedRecords,
+    executionTimeMilliseconds: queryResult.metadata?.grail?.executionTimeMilliseconds,
+    queryId: queryResult.metadata?.grail?.queryId,
+    sampled: queryResult.metadata?.grail?.sampled,
+  };
+
+  console.error(
+    `${logPrefix} scannedBytes=${result.scannedBytes} scannedRecords=${result.scannedRecords} executionTime=${result.executionTimeMilliseconds} queryId=${result.queryId}`,
+  );
+
+  return result;
+};
+
 /**
  * Execute a DQL statement against the Dynatrace API.
  * If the result is immediately available, it will be returned.
  * If the result is not immediately available, it will poll for the result until it is available.
  * @param dtClient
  * @param body - Contains the DQL statement to execute, and optional parameters like maxResultRecords and maxResultBytes
- * @returns the result without metadata and without notifications, or undefined if the query failed or no result was returned.
+ * @returns the result with records, metadata and cost information, or undefined if the query failed or no result was returned.
  */
 export const executeDql = async (
   dtClient: HttpClient,
   body: ExecuteRequest,
-): Promise<QueryResult['records'] | undefined> => {
+): Promise<DqlExecutionResult | undefined> => {
+  // create a Dynatrace QueryExecutionClient
   const queryExecutionClient = new QueryExecutionClient(dtClient);
 
+  // and execute the query (part of body)
   const response = await queryExecutionClient.queryExecute({
     body,
+    // define a dedicated user agent to enable tracking of DQL queries executed by the dynatrace-mcp-server
     dtClientContext: getUserAgent(),
   });
 
+  // check if we already got a result back
   if (response.result) {
-    // return response result immediately
-    return response.result.records;
+    // yes - return response result immediately
+    return createResultAndLog(response.result, 'execute_dql - Metadata:');
   }
-  // else: We might have to poll
+
+  // no result yet? we have wait and poll (this requires requestToken to be set)
   if (response.requestToken) {
     // poll for the result
     let pollResponse;
@@ -48,12 +88,22 @@ export const executeDql = async (
         requestToken: response.requestToken,
         dtClientContext: getUserAgent(),
       });
-      // done - let's return it
+
+      // check if we got a result from the polling endpoint
       if (pollResponse.result) {
-        return pollResponse.result.records;
+        // yes - let's return the polled result
+        return createResultAndLog(pollResponse.result, 'execute_dql Metadata (polled):');
       }
     } while (pollResponse.state === 'RUNNING' || pollResponse.state === 'NOT_STARTED');
+
+    // state != RUNNING and != NOT_STARTED - we should log that
+    console.error(
+      `execute_dql with requestToken ${response.requestToken} ended with state ${pollResponse.state}, stopping...`,
+    );
+    return undefined;
   }
-  // else: whatever happened - we have an error
+
+  // no requestToken set? This should not happen, but just in case, let's log it
+  console.error(`execute_dql did not respond with a requestToken, stopping...`);
   return undefined;
 };

src/capabilities/find-monitored-entity-by-name.ts

@@ -38,10 +38,10 @@ export const findMonitoredEntityByName = async (dtClient: HttpClient, entityName
   // Note: This may be slow, as we are appending multiple entity types above
   const dqlResponse = await executeDql(dtClient, { query: dql });
 
-  if (dqlResponse && dqlResponse.length > 0) {
+  if (dqlResponse && dqlResponse.records && dqlResponse.records.length > 0) {
     let resp = 'The following monitored entities were found:\n';
     // iterate over dqlResponse and create a string with the entity names
-    dqlResponse.forEach((entity) => {
+    dqlResponse.records.forEach((entity) => {
       if (entity) {
         resp += `- Entity '${entity['entity.name']}' of type '${entity['entity.type']} has entity id '${entity.id}'\n`;
       }

src/capabilities/get-monitored-entity-details.ts

@@ -31,19 +31,19 @@ export const getMonitoredEntityDetails = async (
   const dqlResponse = await executeDql(dtClient, { query: dql });
 
   // verify response and length
-  if (!dqlResponse || dqlResponse.length === 0) {
+  if (!dqlResponse || !dqlResponse.records || dqlResponse.records.length === 0) {
     console.error(`No entity found for ID: ${entityId}`);
     return;
   }
 
   // in case we have more than one entity -> log it
-  if (dqlResponse.length > 1) {
+  if (dqlResponse.records.length > 1) {
     console.error(
-      `Multiple entities (${dqlResponse.length}) found for entity ID: ${entityId}. Returning the first one.`,
+      `Multiple entities (${dqlResponse.records.length}) found for entity ID: ${entityId}. Returning the first one.`,
     );
   }
 
-  const entity = dqlResponse[0];
+  const entity = dqlResponse.records[0];
   // make typescript happy; entity should never be null though
   if (!entity) {
     console.error(`No entity found for ID: ${entityId}`);

src/capabilities/list-vulnerabilities.ts

@@ -23,11 +23,11 @@ export const listVulnerabilities = async (dtClient: HttpClient, additionalFilter
     maxResultBytes: 5_000_000, // 5 MB
   });
 
-  if (!response || response.length === 0) {
+  if (!response || !response.records || response.records.length === 0) {
     return [];
   }
 
-  const vulnerabilities = response.map((vuln: any) => {
+  const vulnerabilities = response.records.map((vuln: any) => {
     const vulnerabilityId = vuln['vulnerability.id'] || 'N/A';
     const vulnerabilityDisplayId = vuln['vulnerability.display_id'] || 'N/A';
     const riskScore = vuln['vulnerability.risk.score'] || 'N/A';

src/index.ts

@@ -283,10 +283,10 @@ const main = async () => {
       );
       // get problems (uses fetch)
       const result = await listProblems(dtClient, additionalFilter);
-      if (result && result.length > 0) {
-        let resp = `Found ${result.length} problems! Displaying the top ${maxProblemsToDisplay} problems:\n`;
+      if (result && result.records && result.records.length > 0) {
+        let resp = `Found ${result.records.length} problems! Displaying the top ${maxProblemsToDisplay} problems:\n`;
         // iterate over dqlResponse and create a string with the problem details, but only show the top maxProblemsToDisplay problems
-        result.slice(0, maxProblemsToDisplay).forEach((problem) => {
+        result.records.slice(0, maxProblemsToDisplay).forEach((problem) => {
           if (problem) {
             resp += `Problem ${problem['display_id']} (please refer to this problem with \`problemId\` or \`event.id\` ${problem['problem_id']}))
                   with event.status ${problem['event.status']}, event.category ${problem['event.category']}: ${problem['event.name']} -
@@ -462,7 +462,41 @@ const main = async () => {
       );
       const response = await executeDql(dtClient, { query: dqlStatement });
 
-      return `DQL Response: ${JSON.stringify(response)}`;
+      if (!response) {
+        return 'DQL execution failed or returned no result.';
+      }
+
+      let result = `📊 **DQL Query Results**\n\n`;
+
+      // Cost and Performance Information
+      if (response.scannedRecords !== undefined) {
+        result += `- **Scanned Records:** ${response.scannedRecords.toLocaleString()}\n`;
+      }
+
+      if (response.scannedBytes !== undefined) {
+        const scannedGB = response.scannedBytes / (1000 * 1000 * 1000);
+        result += `- **Scanned Bytes:** ${scannedGB.toFixed(2)} GB`;
+
+        // Cost warning based on scanned bytes
+        if (scannedGB > 500) {
+          result += `\n    ⚠️ **Very High Data Usage Warning:** This query scanned ${scannedGB.toFixed(1)} GB of data, which may impact your Dynatrace consumption. Please take measures to optimize your query, like limiting the timeframe or selecting a bucket.\n`;
+        } else if (scannedGB > 50) {
+          result += `\n    ⚠️ **High Data Usage Warning:** This query scanned ${scannedGB.toFixed(2)} GB of data, which may impact your Dynatrace consumption.\n`;
+        } else if (scannedGB > 5) {
+          result += `\n    💡 **Moderate Data Usage:** This query scanned ${scannedGB.toFixed(2)} GB of data.\n`;
+        } else if (response.scannedBytes === 0) {
+          result += `\n    💡 **No Data consumed:** This query did not consume any data.\n`;
+        }
+      }
+
+      if (response.sampled !== undefined && response.sampled) {
+        result += `- **⚠️ Sampling Used:** Yes (results may be approximate)\n`;
+      }
+
+      result += `\n📋 **Query Results**: (${response.records?.length || 0} records):\n\n`;
+      result += `\`\`\`json\n${JSON.stringify(response.records, null, 2)}\n\`\`\``;
+
+      return result;
     },
   );