diff --git a/src/extension/completions-core/vscode-node/extension/src/ghostText/ghostTextProvider.ts b/src/extension/completions-core/vscode-node/extension/src/ghostText/ghostTextProvider.ts index 9bdddd9328..e6b7a64107 100644 --- a/src/extension/completions-core/vscode-node/extension/src/ghostText/ghostTextProvider.ts +++ b/src/extension/completions-core/vscode-node/extension/src/ghostText/ghostTextProvider.ts @@ -83,6 +83,7 @@ export class GhostTextProvider { opportunityId, }, logContext, + telemetryBuilder.nesBuilder, ); if (!rawCompletions) { diff --git a/src/extension/completions-core/vscode-node/extension/src/vscodeInlineCompletionItemProvider.ts b/src/extension/completions-core/vscode-node/extension/src/vscodeInlineCompletionItemProvider.ts index 48cc2fab5f..4b9dbb893b 100644 --- a/src/extension/completions-core/vscode-node/extension/src/vscodeInlineCompletionItemProvider.ts +++ b/src/extension/completions-core/vscode-node/extension/src/vscodeInlineCompletionItemProvider.ts @@ -173,6 +173,7 @@ export class CopilotInlineCompletionItemProvider extends Disposable implements I handleDidShowCompletionItem(item: GhostTextCompletionItem) { try { + item.telemetryBuilder.setAsShown(); this.copilotCompletionFeedbackTracker.trackItem(item); return this.ghostTextProvider.handleDidShowCompletionItem(item); } catch (e) { diff --git a/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts b/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts index 6a2529a84d..96e580b742 100644 --- a/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts +++ b/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts @@ -8,6 +8,7 @@ import { ITelemetryService } from '../../../../../../platform/telemetry/common/t import { createSha256Hash } from '../../../../../../util/common/crypto'; import { generateUuid } from '../../../../../../util/vs/base/common/uuid'; import { IInstantiationService, ServicesAccessor } from '../../../../../../util/vs/platform/instantiation/common/instantiation'; +import { LlmNESTelemetryBuilder } from '../../../../../inlineEdits/node/nextEditProviderTelemetry'; import { GhostTextLogContext } from '../../../../common/ghostTextContext'; import { initializeTokenizers } from '../../../prompt/src/tokenization'; import { CancellationTokenSource, CancellationToken as ICancellationToken } from '../../../types/src'; @@ -145,6 +146,7 @@ export class GhostTextComputer { token: ICancellationToken | undefined, options: Partial, logContext: GhostTextLogContext, + telemetryBuilder: LlmNESTelemetryBuilder, ): Promise> { const id = generateUuid(); this.currentGhostText.currentRequestId = id; @@ -164,7 +166,7 @@ export class GhostTextComputer { options ); this.notifierService.notifyRequest(completionState, id, telemetryData, token, options); - const result = await this.getGhostTextWithoutAbortHandling(completionState, id, telemetryData, token, options, logContext); + const result = await this.getGhostTextWithoutAbortHandling(completionState, id, telemetryData, token, options, logContext, telemetryBuilder); const statistics = this.contextproviderStatistics.getStatisticsForCompletion(id); const opportunityId = options?.opportunityId ?? 'unknown'; for (const [providerId, statistic] of statistics.getAllUsageStatistics()) { @@ -219,6 +221,7 @@ export class GhostTextComputer { cancellationToken: ICancellationToken | undefined, options: Partial, logContext: GhostTextLogContext, + telemetryBuilder: LlmNESTelemetryBuilder, ): Promise> { let start = preIssuedTelemetryDataWithExp.issuedTime; // Start before getting exp assignments const performanceMetrics: [string, number][] = []; @@ -436,6 +439,10 @@ export class GhostTextComputer { .filter(c => c !== undefined); } + if (choices && choices[1] === ResultType.Cache) { + telemetryBuilder.setIsFromCache(); + } + if (choices !== undefined && choices[0].length === 0) { this.logger.debug(`Found empty inline suggestions locally via ${resultTypeToString(choices[1])}`); return { @@ -632,10 +639,11 @@ export async function getGhostText( token: ICancellationToken | undefined, options: Partial, logContext: GhostTextLogContext, + telemetryBuilder: LlmNESTelemetryBuilder, ): Promise> { const instaService = accessor.get(IInstantiationService); const ghostTextComputer = instaService.createInstance(GhostTextComputer); - return ghostTextComputer.getGhostText(completionState, token, options, logContext); + return ghostTextComputer.getGhostText(completionState, token, options, logContext, telemetryBuilder); } /** diff --git a/src/extension/completions-core/vscode-node/lib/src/ghostText/test/ghostText.test.ts b/src/extension/completions-core/vscode-node/lib/src/ghostText/test/ghostText.test.ts index b5f55c8ae0..75c53313b2 100644 --- a/src/extension/completions-core/vscode-node/lib/src/ghostText/test/ghostText.test.ts +++ b/src/extension/completions-core/vscode-node/lib/src/ghostText/test/ghostText.test.ts @@ -34,6 +34,7 @@ import { ICompletionsCurrentGhostText } from '../current'; import { getGhostText, GhostCompletion } from '../ghostText'; import { ResultType } from '../resultType'; import { mkBasicResultTelemetry } from '../telemetry'; +import { LlmNESTelemetryBuilder } from '../../../../../../inlineEdits/node/nextEditProviderTelemetry'; // Unit tests for ghostText that do not require network connectivity. For other // tests, see lib/e2e/src/ghostText.test.ts. @@ -66,7 +67,8 @@ suite('Isolated GhostText tests', function () { // Setup closures with the state as default function requestGhostText(completionState = state) { - return getGhostText(accessor, completionState, token, {}, new GhostTextLogContext(filePath, doc.version, undefined)); + const telemetryBuilder = new LlmNESTelemetryBuilder(undefined, undefined, undefined, 'ghostText', undefined); + return getGhostText(accessor, completionState, token, {}, new GhostTextLogContext(filePath, doc.version, undefined), telemetryBuilder); } async function requestPrompt(completionState = state) { const telemExp = TelemetryWithExp.createEmptyConfigForTesting(); @@ -653,7 +655,8 @@ suite('Isolated GhostText tests', function () { configProvider.setConfig(ConfigKey.AlwaysRequestMultiline, true); currentGhostText.hasAcceptedCurrentCompletion = () => true; - const response = await getGhostText(accessor, state, undefined, { isSpeculative: true }, new GhostTextLogContext('file:///fizzbuzz.go', doc.version, undefined)); + const telemetryBuilder = new LlmNESTelemetryBuilder(undefined, undefined, undefined, 'ghostText', undefined); + const response = await getGhostText(accessor, state, undefined, { isSpeculative: true }, new GhostTextLogContext('file:///fizzbuzz.go', doc.version, undefined), telemetryBuilder); assert.strictEqual(response.type, 'success'); assert.strictEqual(response.value[0].length, 1); diff --git a/src/extension/completions-core/vscode-node/lib/src/inlineCompletion.ts b/src/extension/completions-core/vscode-node/lib/src/inlineCompletion.ts index 1f1ad89c53..f99a58e618 100644 --- a/src/extension/completions-core/vscode-node/lib/src/inlineCompletion.ts +++ b/src/extension/completions-core/vscode-node/lib/src/inlineCompletion.ts @@ -15,6 +15,7 @@ import { ICompletionsSpeculativeRequestCache } from './ghostText/speculativeRequ import { GhostTextResultWithTelemetry, handleGhostTextResultTelemetry, logger } from './ghostText/telemetry'; import { ICompletionsLogTargetService } from './logger'; import { ITextDocument, TextDocumentContents } from './textDocument'; +import { LlmNESTelemetryBuilder } from '../../../../inlineEdits/node/nextEditProviderTelemetry'; type GetInlineCompletionsOptions = Partial & { formattingOptions?: ITextEditorOptions; @@ -34,10 +35,11 @@ export class GhostText { token: CancellationToken, options: Exclude, 'promptOnly'> = {}, logContext: GhostTextLogContext, + telemetryBuilder: LlmNESTelemetryBuilder, ): Promise { logCompletionLocation(this.logTargetService, textDocument, position); - const result = await this.getInlineCompletionsResult(createCompletionState(textDocument, position), token, options, logContext); + const result = await this.getInlineCompletionsResult(createCompletionState(textDocument, position), token, options, logContext, telemetryBuilder); return this.instantiationService.invokeFunction(handleGhostTextResultTelemetry, result); } @@ -46,6 +48,7 @@ export class GhostText { token: CancellationToken, options: GetInlineCompletionsOptions = {}, logContext: GhostTextLogContext, + telemetryBuilder: LlmNESTelemetryBuilder, ): Promise> { let lineLengthIncrease = 0; // The golang.go extension (and quite possibly others) uses snippets for function completions, which collapse down @@ -56,8 +59,12 @@ export class GhostText { lineLengthIncrease = completionState.position.character - options.selectedCompletionInfo.range.end.character; } - const result = await this.instantiationService.invokeFunction(getGhostText, completionState, token, options, logContext); - if (result.type !== 'success') { return result; } + const result = await this.instantiationService.invokeFunction(getGhostText, completionState, token, options, logContext, telemetryBuilder); + + if (result.type !== 'success') { + return result; + } + const [resultArray, resultType] = result.value; if (token.isCancellationRequested) { @@ -95,7 +102,7 @@ export class GhostText { // Cache speculative request to be triggered when telemetryShown is called const specOpts = { isSpeculative: true, opportunityId: options.opportunityId }; - const fn = () => this.instantiationService.invokeFunction(getGhostText, completionState, undefined, specOpts, logContext); + const fn = () => this.instantiationService.invokeFunction(getGhostText, completionState, undefined, specOpts, logContext, telemetryBuilder); this.speculativeRequestCache.set(completions[0].clientCompletionId, fn); } diff --git a/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.ts b/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.ts index 6126c459a4..2a1bc1fda3 100644 --- a/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.ts +++ b/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.ts @@ -12,6 +12,7 @@ import { IPosition, ITextDocument } from '../../textDocument'; import { ICompletionsContextProviderBridgeService } from '../components/contextProviderBridge'; import { extractPrompt, ExtractPromptOptions } from '../prompt'; import { GhostTextLogContext } from '../../../../../common/ghostTextContext'; +import { LlmNESTelemetryBuilder } from '../../../../../../inlineEdits/node/nextEditProviderTelemetry'; export async function extractPromptInternal( accessor: ServicesAccessor, @@ -33,5 +34,6 @@ export async function getGhostTextInternal( position: IPosition, token?: CancellationToken ) { - return getGhostText(accessor, createCompletionState(textDocument, position), token, { opportunityId: 'opId' }, new GhostTextLogContext(textDocument.uri, textDocument.version, undefined)); + const telemetryBuilder = new LlmNESTelemetryBuilder(undefined, undefined, undefined, 'ghostText', undefined); + return getGhostText(accessor, createCompletionState(textDocument, position), token, { opportunityId: 'opId' }, new GhostTextLogContext(textDocument.uri, textDocument.version, undefined), telemetryBuilder); } diff --git a/src/extension/completions-core/vscode-node/lib/src/test/inlineCompletion.test.ts b/src/extension/completions-core/vscode-node/lib/src/test/inlineCompletion.test.ts index 5e75e935d2..5c556fb070 100644 --- a/src/extension/completions-core/vscode-node/lib/src/test/inlineCompletion.test.ts +++ b/src/extension/completions-core/vscode-node/lib/src/test/inlineCompletion.test.ts @@ -20,6 +20,7 @@ import { createLibTestingContext } from './context'; import { createFakeCompletionResponse, StaticFetcher } from './fetcher'; import { withInMemoryTelemetry } from './telemetry'; import { createTextDocument } from './textDocument'; +import { LlmNESTelemetryBuilder } from '../../../../../inlineEdits/node/nextEditProviderTelemetry'; suite('getInlineCompletions()', function () { function setupCompletion( @@ -38,7 +39,8 @@ suite('getInlineCompletions()', function () { function requestInlineCompletions(textDoc = doc, pos = position) { const instaService = accessor.get(IInstantiationService); const ghostText = instaService.createInstance(GhostText); - return ghostText.getInlineCompletions(textDoc, pos, CancellationToken.None, undefined, new GhostTextLogContext(textDoc.uri, textDoc.version, undefined)); + const telemetryBuilder = new LlmNESTelemetryBuilder(undefined, undefined, undefined, 'ghostText', undefined); + return ghostText.getInlineCompletions(textDoc, pos, CancellationToken.None, undefined, new GhostTextLogContext(textDoc.uri, textDoc.version, undefined), telemetryBuilder); } return { diff --git a/src/lib/node/chatLibMain.ts b/src/lib/node/chatLibMain.ts index ee3ae46fe9..455098cbeb 100644 --- a/src/lib/node/chatLibMain.ts +++ b/src/lib/node/chatLibMain.ts @@ -662,7 +662,8 @@ class InlineCompletionsProvider extends Disposable implements IInlineCompletions } async getInlineCompletions(textDocument: ITextDocument, position: Position, token?: CancellationToken, options?: IGetInlineCompletionsOptions): Promise { - return await this.ghostText.getInlineCompletions(textDocument, position, token ?? CancellationToken.None, options, new GhostTextLogContext(textDocument.uri, textDocument.version, undefined)); + const telemetryBuilder = new LlmNESTelemetryBuilder(undefined, undefined, undefined, 'ghostText', undefined); + return await this.ghostText.getInlineCompletions(textDocument, position, token ?? CancellationToken.None, options, new GhostTextLogContext(textDocument.uri, textDocument.version, undefined), telemetryBuilder); } async inlineCompletionShown(completionId: string): Promise {