Report when AI agent suggestions are shown to the user.
This change introduces a new reporting interaction type,
AI_AGENT_SUGGESTIONS_SHOWN, and an associated AiAgentEventDetails type.
The gemini-message component now reports this interaction when an AI
agent's response is fully rendered.
Google-Bug-Id: b/489833285
Release-Notes: skip
Change-Id: Ib9f9f5f2b16681425104d39cab43e5181c2ef2df
diff --git a/polygerrit-ui/app/constants/reporting.ts b/polygerrit-ui/app/constants/reporting.ts
index f95148b..bae1c58 100644
--- a/polygerrit-ui/app/constants/reporting.ts
+++ b/polygerrit-ui/app/constants/reporting.ts
@@ -169,4 +169,21 @@
COMMENT_COMPLETION_SUGGESTION_FETCHED = 'comment-completion-suggestion-fetched',
COPY_AI_PROMPT = 'copy-ai-prompt',
+
+ // AI agent suggests comments/fixes to user.
+ AI_AGENT_SUGGESTIONS_SHOWN = 'ai-agent-suggestions-shown',
}
+
+/**
+ * EventDetails to be passed to the reportInteraction method for AI agent
+ * interactions.
+ */
+export type AiAgentEventDetails = {
+ host: string;
+ agentId: string;
+ conversationId: string;
+ // Each agent response in a conversation is a turn.
+ turnIndex: number;
+ // commentCount is 0 if agent ran but didn't suggest any comments/fixes.
+ commentCount: number;
+};
diff --git a/polygerrit-ui/app/elements/chat-panel/gemini-message.ts b/polygerrit-ui/app/elements/chat-panel/gemini-message.ts
index 5f568ff..8f0dbee 100644
--- a/polygerrit-ui/app/elements/chat-panel/gemini-message.ts
+++ b/polygerrit-ui/app/elements/chat-panel/gemini-message.ts
@@ -13,10 +13,11 @@
import './references-dropdown';
import './message-actions';
-import {css, html, LitElement} from 'lit';
+import {css, html, LitElement, PropertyValues} from 'lit';
import {customElement, property, state} from 'lit/decorators.js';
import {when} from 'lit/directives/when.js';
+import {AiAgentEventDetails, Interaction} from '../../constants/reporting';
import {changeModelToken} from '../../models/change/change-model';
import {
filesModelToken,
@@ -31,6 +32,7 @@
} from '../../models/chat/chat-model';
import {commentsModelToken} from '../../models/comments/comments-model';
import {resolve} from '../../models/dependency';
+import {getAppContext} from '../../services/app-context';
import {NumericChangeId, PatchSetNumber} from '../../types/common';
import {
compareComments,
@@ -64,6 +66,10 @@
@state() latestPatchNum?: PatchSetNumber;
+ @state() private conversationId?: string;
+
+ private reportedSuggestionsShown = false;
+
private readonly getChatModel = resolve(this, chatModelToken);
private readonly getCommentsModel = resolve(this, commentsModelToken);
@@ -72,6 +78,8 @@
private readonly getFilesModel = resolve(this, filesModelToken);
+ private readonly reportingService = getAppContext().reportingService;
+
static override styles = [
materialStyles,
css`
@@ -212,6 +220,11 @@
() => this.getChangeModel().latestPatchNum$,
x => (this.latestPatchNum = x)
);
+ subscribe(
+ this,
+ () => this.getChatModel().conversationId$,
+ x => (this.conversationId = x)
+ );
}
private async onAddAsComment(part: CreateCommentPart) {
@@ -242,6 +255,17 @@
fire(this, 'open-diff-in-change-view', {path, lineNum});
}
+ override updated(changedProperties: PropertyValues) {
+ if (changedProperties.has('turns') && !this.reportedSuggestionsShown) {
+ if (
+ this.turnIndex < this.turns.length &&
+ this.message()?.responseComplete
+ ) {
+ this.reportSuggestionsShown();
+ }
+ }
+ }
+
override render() {
if (this.turnIndex >= this.turns.length) return;
const message = this.message();
@@ -405,10 +429,9 @@
}
private sortedComments() {
- return this.message()
- .responseParts.filter(
- part => part.type === ResponsePartType.CREATE_COMMENT
- )
+ const parts = this.message()?.responseParts ?? [];
+ return parts
+ .filter(part => part.type === ResponsePartType.CREATE_COMMENT)
.sort((p1, p2) => {
const c1 = {...createNew(p1.comment.message), ...p1.comment};
const c2 = {...createNew(p2.comment.message), ...p2.comment};
@@ -419,9 +442,27 @@
private turnId() {
return {
turnIndex: this.turnIndex,
- regenerationIndex: this.message().regenerationIndex,
+ regenerationIndex: this.message()?.regenerationIndex ?? 0,
};
}
+
+ private reportSuggestionsShown() {
+ if (!this.conversationId) return;
+ this.reportedSuggestionsShown = true;
+
+ const agentId = this.turns[this.turnIndex]?.userMessage?.actionId ?? '';
+ const details: AiAgentEventDetails = {
+ host: window.location.host,
+ agentId,
+ conversationId: this.conversationId,
+ turnIndex: this.turnIndex,
+ commentCount: this.sortedComments().length,
+ };
+ this.reportingService.reportInteraction(
+ Interaction.AI_AGENT_SUGGESTIONS_SHOWN,
+ details
+ );
+ }
}
declare global {
diff --git a/polygerrit-ui/app/elements/chat-panel/gemini-message_test.ts b/polygerrit-ui/app/elements/chat-panel/gemini-message_test.ts
index dcf2904..57401f3 100644
--- a/polygerrit-ui/app/elements/chat-panel/gemini-message_test.ts
+++ b/polygerrit-ui/app/elements/chat-panel/gemini-message_test.ts
@@ -12,6 +12,7 @@
CreateCommentPart,
GeminiMessage as GeminiMessageModel,
ResponsePartType,
+ TextPart,
Turn,
UserType,
} from '../../models/chat/chat-model';
@@ -25,6 +26,8 @@
import {chatProvider, createChange} from '../../test/test-data-generators';
import {ParsedChangeInfo} from '../../types/types';
import {CommentsModel} from '../../models/comments/comments-model';
+import {AiAgentEventDetails, Interaction} from '../../constants/reporting';
+import {getAppContext} from '../../services/app-context';
suite('gemini-message tests', () => {
let element: GeminiMessage;
@@ -71,6 +74,22 @@
};
}
+ const RESPONSE_TEXT: TextPart = {
+ id: 0,
+ type: ResponsePartType.TEXT,
+ content: 'test message',
+ };
+ const RESPONSE_CREATE_COMMENT: CreateCommentPart = {
+ id: 1,
+ type: ResponsePartType.CREATE_COMMENT,
+ content: 'test comment',
+ commentCreationId: 'test-id',
+ comment: {
+ message: 'test comment',
+ path: '/test/path',
+ },
+ };
+
test('renders thinking', async () => {
const turn = createTurn({responseComplete: false});
chatModel.updateState({...chatModel.getState(), turns: [turn]});
@@ -113,9 +132,7 @@
test('renders text response', async () => {
const turn = createTurn({
responseComplete: true,
- responseParts: [
- {id: 0, type: ResponsePartType.TEXT, content: 'test message'},
- ],
+ responseParts: [RESPONSE_TEXT],
});
chatModel.updateState({...chatModel.getState(), turns: [turn]});
await element.updateComplete;
@@ -151,19 +168,9 @@
});
test('renders suggested comment', async () => {
- const comment: CreateCommentPart = {
- id: 1,
- type: ResponsePartType.CREATE_COMMENT,
- content: 'test comment',
- commentCreationId: 'test-id',
- comment: {
- message: 'test comment',
- path: '/test/path',
- },
- };
const turn = createTurn({
responseComplete: true,
- responseParts: [comment],
+ responseParts: [RESPONSE_CREATE_COMMENT],
});
chatModel.updateState({...chatModel.getState(), turns: [turn]});
await element.updateComplete;
@@ -190,9 +197,7 @@
test('renders citations', async () => {
const turn = createTurn({
responseComplete: true,
- responseParts: [
- {id: 0, type: ResponsePartType.TEXT, content: 'test message'},
- ],
+ responseParts: [RESPONSE_TEXT],
citations: ['http://example.com'],
});
chatModel.updateState({...chatModel.getState(), turns: [turn]});
@@ -213,9 +218,7 @@
];
const turn = createTurn({
responseComplete: true,
- responseParts: [
- {id: 0, type: ResponsePartType.TEXT, content: 'test message'},
- ],
+ responseParts: [RESPONSE_TEXT],
references,
});
chatModel.updateState({...chatModel.getState(), turns: [turn]});
@@ -227,4 +230,40 @@
);
assert.isOk(referencesDropdown);
});
+
+ test('reports AI_AGENT_SUGGESTIONS_SHOWN interaction', async () => {
+ chatModel.updateState({
+ ...chatModel.getState(),
+ id: 'test-conversation-id',
+ selectedModelId: 'gemini-model-id',
+ });
+
+ const reportStub = sinon.stub(
+ getAppContext().reportingService,
+ 'reportInteraction'
+ );
+
+ const turn = createTurn({
+ responseComplete: true,
+ responseParts: [RESPONSE_TEXT, RESPONSE_CREATE_COMMENT],
+ });
+ const updatedTurn = {
+ ...turn,
+ userMessage: {...turn.userMessage, actionId: 'custom-agent-id'},
+ };
+
+ chatModel.updateState({...chatModel.getState(), turns: [updatedTurn]});
+ await element.updateComplete;
+
+ assert.isTrue(reportStub.calledOnce);
+ assert.equal(
+ reportStub.firstCall.args[0],
+ Interaction.AI_AGENT_SUGGESTIONS_SHOWN
+ );
+ const details = reportStub.firstCall.args[1] as AiAgentEventDetails;
+ assert.equal(details.host, window.location.host);
+ assert.equal(details.conversationId, 'test-conversation-id');
+ assert.equal(details.agentId, 'custom-agent-id');
+ assert.equal(details.commentCount, 1);
+ });
});
