Merge "Instrument telemetry for AI Review Agent chat requests"
diff --git a/polygerrit-ui/app/constants/reporting.ts b/polygerrit-ui/app/constants/reporting.ts
index 5502a73..1b3c5c0 100644
--- a/polygerrit-ui/app/constants/reporting.ts
+++ b/polygerrit-ui/app/constants/reporting.ts
@@ -103,6 +103,8 @@
COPY_TO_CLIPBOARD = 'CopyToClipboard',
// Time to autocomplete a comment
COMMENT_COMPLETION = 'CommentCompletion',
+ // Time for AI chat requests to complete
+ AI_CHAT_REQUEST = 'AiChatRequest',
}
export enum Interaction {
@@ -184,6 +186,8 @@
FLOWS_TAB_RENDERED = 'flows-tab-rendered',
CREATE_FLOW_DIALOG_OPENED = 'create-flow-dialog-opened',
FLOW_CREATED = 'flow-created',
+ // AI Chat interaction request failures
+ AI_CHAT_FAILURE = 'ai-chat-failure',
}
/**
diff --git a/polygerrit-ui/app/models/chat/chat-model.ts b/polygerrit-ui/app/models/chat/chat-model.ts
index 619424b..2711edb 100644
--- a/polygerrit-ui/app/models/chat/chat-model.ts
+++ b/polygerrit-ui/app/models/chat/chat-model.ts
@@ -37,6 +37,8 @@
import {contextItemEquals} from './context-item-util';
import {FilesModel, NormalizedFileInfo} from '../change/files-model';
import {isMagicPath} from '../../utils/path-list-util';
+import {getAppContext} from '../../services/app-context';
+import {Interaction, Timing} from '../../constants/reporting';
/** The available display modes in the chat panel. */
export enum ChatPanelMode {
@@ -616,6 +618,19 @@
});
},
emitError: (errorMessage: string) => {
+ getAppContext().reportingService.timeEnd(Timing.AI_CHAT_REQUEST, {
+ modelName: request.model_name,
+ actionId: action.id,
+ error: errorMessage,
+ });
+ getAppContext().reportingService.reportInteraction(
+ Interaction.AI_CHAT_FAILURE,
+ {
+ modelName: request.model_name,
+ actionId: action.id,
+ error: errorMessage,
+ }
+ );
const state = this.getState();
if (state.id !== conversationId) return;
const turns: readonly Turn[] = state.turns;
@@ -630,6 +645,10 @@
});
},
done: () => {
+ getAppContext().reportingService.timeEnd(Timing.AI_CHAT_REQUEST, {
+ modelName: request.model_name,
+ actionId: action.id,
+ });
const state = this.getState();
if (state.id !== conversationId) return;
assert(turnIndex < state.turns.length, 'turn index out of bounds');
@@ -642,6 +661,7 @@
});
},
};
+ getAppContext().reportingService.time(Timing.AI_CHAT_REQUEST);
this.plugin?.chat?.(request, listener);
}
diff --git a/polygerrit-ui/app/models/chat/chat-model_test.ts b/polygerrit-ui/app/models/chat/chat-model_test.ts
index 9f70830..d244bd4 100644
--- a/polygerrit-ui/app/models/chat/chat-model_test.ts
+++ b/polygerrit-ui/app/models/chat/chat-model_test.ts
@@ -16,6 +16,8 @@
import sinon from 'sinon';
import {ParsedChangeInfo} from '../../types/types';
+import {getAppContext} from '../../services/app-context';
+import {Interaction, Timing} from '../../constants/reporting';
suite('chat-model tests', () => {
let model: ChatModel;
@@ -345,4 +347,99 @@
const state = model.getState();
assert.equal(state.turns[0].geminiMessage.regenerationIndex, 0);
});
+
+ suite('telemetry reporting', () => {
+ let timeStub: sinon.SinonStub;
+ let timeEndStub: sinon.SinonStub;
+ let reportInteractionStub: sinon.SinonStub;
+
+ setup(() => {
+ timeStub = sinon.stub(getAppContext().reportingService, 'time');
+ timeEndStub = sinon.stub(getAppContext().reportingService, 'timeEnd');
+ reportInteractionStub = sinon.stub(
+ getAppContext().reportingService,
+ 'reportInteraction'
+ );
+
+ // Set up a change, models, and actions
+ const models = {
+ models: [
+ {
+ model_id: 'test-model',
+ full_display_text: 'Test Model',
+ short_text: 'Test',
+ },
+ ],
+ default_model_id: 'test-model',
+ };
+ const actions = {
+ actions: [
+ {
+ id: 'test-action',
+ display_text: 'Test Action',
+ initial_user_prompt: 'Test Prompt',
+ },
+ ],
+ default_action_id: 'test-action',
+ };
+ (provider.getActions as sinon.SinonStub).resolves(actions);
+ (provider.getModels as sinon.SinonStub).resolves(models);
+
+ changeModel.updateStateChange(createParsedChange());
+ });
+
+ test('chat request starts a timer', async () => {
+ await new Promise(resolve => setTimeout(resolve, 0));
+
+ model.updateUserInput('hello');
+ model.chat('hello', 'test-action', 0);
+
+ assert.isTrue(timeStub.calledOnceWith(Timing.AI_CHAT_REQUEST));
+ });
+
+ test('chat request success stops the timer', async () => {
+ await new Promise(resolve => setTimeout(resolve, 0));
+
+ (provider.chat as sinon.SinonStub).callsFake((_, listener) => {
+ listener.done();
+ });
+
+ model.updateUserInput('hello');
+ model.chat('hello', 'test-action', 0);
+
+ assert.isTrue(
+ timeEndStub.calledOnceWith(Timing.AI_CHAT_REQUEST, {
+ modelName: 'test-model',
+ actionId: 'test-action',
+ })
+ );
+ });
+
+ test('chat request failure stops the timer and logs interaction', async () => {
+ await new Promise(resolve => setTimeout(resolve, 0));
+
+ (provider.chat as sinon.SinonStub).callsFake((_, listener) => {
+ listener.emitError('some error');
+ });
+
+ model.updateUserInput('hello');
+ model.chat('hello', 'test-action', 0);
+
+ assert.isTrue(
+ timeEndStub.calledOnceWith(Timing.AI_CHAT_REQUEST, {
+ modelName: 'test-model',
+ actionId: 'test-action',
+ error: 'some error',
+ })
+ );
+
+ assert.isTrue(
+ reportInteractionStub.calledOnceWith(Interaction.AI_CHAT_FAILURE, {
+ modelName: 'test-model',
+ actionId: 'test-action',
+ error: 'some error',
+ })
+ );
+ });
+ });
});
