Integrate stateful review into testing suite
Stateful patch set review has been incorporated into the testing suite.
After testing, the accuracy of both the thread message and of the review
reply is verified against expected values.
Change-Id: I3a2b0252a55940187f00f1d0679392b677a625f7
Signed-off-by: Patrizio <patrizio.gelosi@amarulasolutions.com>
diff --git a/src/main/java/com/googlesource/gerrit/plugins/chatgpt/mode/stateful/client/api/chatgpt/ChatGptClientStateful.java b/src/main/java/com/googlesource/gerrit/plugins/chatgpt/mode/stateful/client/api/chatgpt/ChatGptClientStateful.java
index 4f46205..c401cd0 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/chatgpt/mode/stateful/client/api/chatgpt/ChatGptClientStateful.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/chatgpt/mode/stateful/client/api/chatgpt/ChatGptClientStateful.java
@@ -35,6 +35,9 @@
ChatGptRun chatGptRun = new ChatGptRun(threadId, config, pluginDataHandler);
chatGptRun.createRun();
chatGptRun.pollRun();
+ // Attribute `requestBody` is valued for testing purposes
+ requestBody = chatGptThread.getAddMessageRequestBody();
+ log.debug("ChatGPT request body: {}", requestBody);
return getResponseContent(chatGptRun.getFirstStep());
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewStatefulTest.java b/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewStatefulTest.java
new file mode 100644
index 0000000..55d8252
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewStatefulTest.java
@@ -0,0 +1,176 @@
+package com.googlesource.gerrit.plugins.chatgpt;
+
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.google.common.net.HttpHeaders;
+import com.google.gerrit.extensions.api.changes.ReviewInput;
+import com.google.gerrit.extensions.restapi.BinaryResult;
+import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gson.JsonObject;
+import com.googlesource.gerrit.plugins.chatgpt.mode.common.model.api.chatgpt.ChatGptResponseContent;
+import com.googlesource.gerrit.plugins.chatgpt.mode.stateful.client.api.UriResourceLocatorStateful;
+import com.googlesource.gerrit.plugins.chatgpt.mode.stateful.client.prompt.ChatGptPromptStateful;
+import com.googlesource.gerrit.plugins.chatgpt.mode.stateful.model.api.chatgpt.ChatGptListResponse;
+import com.googlesource.gerrit.plugins.chatgpt.settings.Settings.MODES;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.entity.ContentType;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+
+
+import static com.googlesource.gerrit.plugins.chatgpt.mode.stateful.client.api.chatgpt.ChatGptRun.COMPLETED_STATUS;
+import static com.googlesource.gerrit.plugins.chatgpt.settings.Settings.GERRIT_PATCH_SET_FILENAME;
+import static com.googlesource.gerrit.plugins.chatgpt.utils.GsonUtils.getGson;
+import static java.net.HttpURLConnection.HTTP_OK;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+@Slf4j
+@RunWith(MockitoJUnitRunner.class)
+public class ChatGptReviewStatefulTest extends ChatGptReviewTestBase {
+ private static final String CHAT_GPT_FILE_ID = "file-TEST_FILE_ID";
+ private static final String CHAT_GPT_ASSISTANT_ID = "asst_TEST_ASSISTANT_ID";
+ private static final String CHAT_GPT_THREAD_ID = "thread_TEST_THREAD_ID";
+ private static final String CHAT_GPT_MESSAGE_ID = "msg_TEST_MESSAGE_ID";
+ private static final String CHAT_GPT_RUN_ID = "run_TEST_RUN_ID";
+
+ private String formattedPatchContent;
+ private String reviewMessage;
+ private ChatGptPromptStateful chatGptPromptStateful;
+ private JsonObject threadMessage;
+
+ public ChatGptReviewStatefulTest() {
+ MockitoAnnotations.openMocks(this);
+ }
+
+ protected void initGlobalAndProjectConfig() {
+ super.initGlobalAndProjectConfig();
+
+ // Mock the Global Config values that differ from the ones provided by Default
+ when(globalConfig.getString(Mockito.eq("gptMode"), Mockito.anyString()))
+ .thenReturn(MODES.stateful.name());
+ }
+
+ protected void initConfig() {
+ super.initConfig();
+
+ // Load the prompts
+ chatGptPromptStateful = new ChatGptPromptStateful(config, getGerritChange());
+ }
+
+ protected void setupMockRequests() throws RestApiException {
+ super.setupMockRequests();
+
+ // Mock the behavior of the Git Repository Manager
+ String repoJson = readTestFile("__files/gitProjectFiles.json");
+ when(gitRepoFiles.getGitRepoFiles(any())).thenReturn(repoJson);
+
+ // Mock the behavior of the ChatGPT create-file request
+ WireMock.stubFor(WireMock.post(WireMock.urlEqualTo(URI.create(config.getGptDomain()
+ + UriResourceLocatorStateful.filesCreateUri()).getPath()))
+ .willReturn(WireMock.aResponse()
+ .withStatus(HTTP_OK)
+ .withHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString())
+ .withBody("{\"id\": " + CHAT_GPT_FILE_ID + "}")));
+
+ // Mock the behavior of the ChatGPT create-assistant request
+ WireMock.stubFor(WireMock.post(WireMock.urlEqualTo(URI.create(config.getGptDomain()
+ + UriResourceLocatorStateful.assistantCreateUri()).getPath()))
+ .willReturn(WireMock.aResponse()
+ .withStatus(HTTP_OK)
+ .withHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString())
+ .withBody("{\"id\": " + CHAT_GPT_ASSISTANT_ID + "}")));
+
+ // Mock the behavior of the ChatGPT create-thread request
+ WireMock.stubFor(WireMock.post(WireMock.urlEqualTo(URI.create(config.getGptDomain()
+ + UriResourceLocatorStateful.threadsUri()).getPath()))
+ .willReturn(WireMock.aResponse()
+ .withStatus(HTTP_OK)
+ .withHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString())
+ .withBody("{\"id\": " + CHAT_GPT_THREAD_ID + "}")));
+
+ // Mock the behavior of the ChatGPT add-message-to-thread request
+ WireMock.stubFor(WireMock.post(WireMock.urlEqualTo(URI.create(config.getGptDomain()
+ + UriResourceLocatorStateful.threadMessagesUri(CHAT_GPT_THREAD_ID)).getPath()))
+ .willReturn(WireMock.aResponse()
+ .withStatus(HTTP_OK)
+ .withHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString())
+ .withBody("{\"id\": " + CHAT_GPT_MESSAGE_ID + "}")));
+
+ // Mock the behavior of the ChatGPT create-run request
+ WireMock.stubFor(WireMock.post(WireMock.urlEqualTo(URI.create(config.getGptDomain()
+ + UriResourceLocatorStateful.runsUri(CHAT_GPT_THREAD_ID)).getPath()))
+ .willReturn(WireMock.aResponse()
+ .withStatus(HTTP_OK)
+ .withHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString())
+ .withBody("{\"id\": " + CHAT_GPT_RUN_ID + "}")));
+
+ // Mock the behavior of the ChatGPT retrieve-run request
+ WireMock.stubFor(WireMock.post(WireMock.urlEqualTo(URI.create(config.getGptDomain()
+ + UriResourceLocatorStateful.runRetrieveUri(CHAT_GPT_THREAD_ID, CHAT_GPT_RUN_ID)).getPath()))
+ .willReturn(WireMock.aResponse()
+ .withStatus(HTTP_OK)
+ .withHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString())
+ .withBody("{\"status\": " + COMPLETED_STATUS + "}")));
+
+ // Mock the behavior of the ChatGPT retrieve-run-steps request
+ WireMock.stubFor(WireMock.get(WireMock.urlEqualTo(URI.create(config.getGptDomain()
+ + UriResourceLocatorStateful.runStepsUri(CHAT_GPT_THREAD_ID, CHAT_GPT_RUN_ID)).getPath()))
+ .willReturn(WireMock.aResponse()
+ .withStatus(HTTP_OK)
+ .withHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString())
+ .withBodyFile("chatGptRunStepsResponse.json")));
+
+ // Mock the behavior of the formatted patch request
+ formattedPatchContent = readTestFile("__files/gerritFormattedPatch.txt");
+ ByteArrayInputStream inputStream = new ByteArrayInputStream(formattedPatchContent.getBytes());
+ BinaryResult binaryResult = BinaryResult.create(inputStream)
+ .setContentType("text/plain")
+ .setContentLength(formattedPatchContent.length());
+ when(revisionApiMock.patch()).thenReturn(binaryResult);
+ }
+
+ protected void initComparisonContent() {
+ super.initComparisonContent();
+
+ reviewMessage = getReviewMessage();
+ }
+
+ protected ArgumentCaptor<ReviewInput> testRequestSent() throws RestApiException {
+ ArgumentCaptor<ReviewInput> reviewInputCaptor = super.testRequestSent();
+ threadMessage = gptRequestBody.getAsJsonObject();
+ return reviewInputCaptor;
+ }
+
+ private String getReviewMessage() {
+ ChatGptListResponse reviewResponse = getGson().fromJson(readTestFile(
+ "__files/chatGptRunStepsResponse.json"
+ ), ChatGptListResponse.class);
+ String reviewJsonResponse = reviewResponse.getData().get(0).getStepDetails().getToolCalls().get(0).getFunction()
+ .getArguments();
+ return getGson().fromJson(reviewJsonResponse, ChatGptResponseContent.class).getReplies().get(0).getReply();
+ }
+
+ @Test
+ public void patchSetCreatedOrUpdated() throws Exception {
+ String reviewUserPrompt = chatGptPromptStateful.getDefaultGptThreadReviewMessage(formattedPatchContent);
+
+ handleEventBasedOnType(false);
+
+ ArgumentCaptor<ReviewInput> captor = testRequestSent();
+ String userPrompt = threadMessage.get("content").getAsString();
+ Assert.assertEquals(reviewUserPrompt, userPrompt);
+ Assert.assertEquals(
+ reviewMessage,
+ captor.getAllValues().get(0).comments.get(GERRIT_PATCH_SET_FILENAME).get(0).message
+ );
+ }
+
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewStatelessTest.java b/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewStatelessTest.java
index 17c4a08..645e0ff 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewStatelessTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewStatelessTest.java
@@ -2,6 +2,7 @@
import com.github.tomakehurst.wiremock.client.WireMock;
import com.google.common.net.HttpHeaders;
+import com.google.gson.JsonArray;
import com.googlesource.gerrit.plugins.chatgpt.listener.EventHandlerTask;
import com.google.gerrit.extensions.api.changes.FileApi;
import com.google.gerrit.extensions.api.changes.ReviewInput;
@@ -41,6 +42,7 @@
private String promptTagComments;
private String diffContent;
private ReviewInput gerritPatchSetReview;
+ private JsonArray prompts;
private ChatGptPromptStateless chatGptPromptStateless;
@@ -51,7 +53,6 @@
.thenReturn(GPT_STREAM_OUTPUT);
when(globalConfig.getBoolean(Mockito.eq("gptReviewCommitMessages"), Mockito.anyBoolean()))
.thenReturn(true);
- when(globalConfig.getString("gerritUserName")).thenReturn(GERRIT_GPT_USERNAME);
super.initConfig();
@@ -62,9 +63,6 @@
protected void setupMockRequests() throws RestApiException {
super.setupMockRequests();
- // Mock the GerritApi's revision API
- when(changeApiMock.current()).thenReturn(revisionApiMock);
-
// Mock the behavior of the gerritPatchSetFiles request
Map<String, FileInfo> files =
readTestFileToType(
@@ -102,6 +100,12 @@
expectedSystemPromptReview = ChatGptPromptStateless.getDefaultGptReviewSystemPrompt();
}
+ protected ArgumentCaptor<ReviewInput> testRequestSent() throws RestApiException {
+ ArgumentCaptor<ReviewInput> reviewInputCaptor = super.testRequestSent();
+ prompts = gptRequestBody.get("messages").getAsJsonArray();
+ return reviewInputCaptor;
+ }
+
private String getReviewUserPrompt() {
return joinWithNewLine(Arrays.asList(
ChatGptPromptStateless.DEFAULT_GPT_REVIEW_PROMPT,
diff --git a/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewTestBase.java b/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewTestBase.java
index aa7e361..fb1859f 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewTestBase.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/chatgpt/ChatGptReviewTestBase.java
@@ -1,7 +1,6 @@
package com.googlesource.gerrit.plugins.chatgpt;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
-import com.github.tomakehurst.wiremock.verification.LoggedRequest;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.BranchNameKey;
import com.google.gerrit.entities.Change;
@@ -25,10 +24,8 @@
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.PatchSetCreatedEvent;
import com.google.gerrit.server.events.PatchSetEvent;
-import com.google.gerrit.server.project.NoSuchProjectException;
import com.google.gerrit.server.util.OneOffRequestContext;
import com.google.gson.Gson;
-import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
@@ -81,14 +78,12 @@
public class ChatGptReviewTestBase {
protected static final Path basePath = Paths.get("src/test/resources");
- protected static final String GERRIT_AUTH_BASE_URL = "http://localhost:9527";
protected static final int GERRIT_GPT_ACCOUNT_ID = 1000000;
protected static final String GERRIT_GPT_USERNAME = "gpt";
protected static final int GERRIT_USER_ACCOUNT_ID = 1000001;
protected static final String GERRIT_USER_ACCOUNT_NAME = "Test";
protected static final String GERRIT_USER_ACCOUNT_EMAIL = "test@example.com";
protected static final String GERRIT_USER_USERNAME = "test";
- protected static final String GERRIT_USER_PASSWORD = "test";
protected static final String GERRIT_USER_GROUP = "Test";
protected static final String GPT_TOKEN = "tk-test";
protected static final String GPT_DOMAIN = "http://localhost:9527";
@@ -131,11 +126,10 @@
protected GerritClient gerritClient;
protected PatchSetReviewer patchSetReviewer;
protected ConfigCreator mockConfigCreator;
- protected List<LoggedRequest> loggedRequests;
- protected JsonArray prompts;
+ protected JsonObject gptRequestBody;
@Before
- public void before() throws NoSuchProjectException, RestApiException {
+ public void before() throws RestApiException {
initGlobalAndProjectConfig();
initConfig();
setupMockRequests();
@@ -165,6 +159,8 @@
// Mock the Global Config values that differ from the ones provided by Default
when(globalConfig.getString(Mockito.eq("gptDomain"), Mockito.anyString()))
.thenReturn(GPT_DOMAIN);
+ when(globalConfig.getString("gerritUserName")).thenReturn(GERRIT_GPT_USERNAME);
+
projectConfig = mock(PluginConfig.class);
@@ -197,6 +193,9 @@
// Mock the behavior of the gerrit Review request
mockGerritReviewApiCall();
+
+ // Mock the GerritApi's revision API
+ when(changeApiMock.current()).thenReturn(revisionApiMock);
}
private Accounts mockGerritAccountsRestEndpoint() {
@@ -205,8 +204,7 @@
return accountsMock;
}
- private void mockGerritAccountsQueryApiCall(
- String username, int expectedAccountId) throws RestApiException {
+ private void mockGerritAccountsQueryApiCall(String username, int expectedAccountId) {
AccountState accountStateMock = mock(AccountState.class);
Account accountMock = mock(Account.class);
when(accountStateMock.account()).thenReturn(accountMock);
@@ -295,9 +293,7 @@
protected ArgumentCaptor<ReviewInput> testRequestSent() throws RestApiException {
ArgumentCaptor<ReviewInput> reviewInputCaptor = ArgumentCaptor.forClass(ReviewInput.class);
verify(revisionApiMock).review(reviewInputCaptor.capture());
- JsonObject gptRequestBody = getGson().fromJson(patchSetReviewer.getChatGptClient().getRequestBody(),
- JsonObject.class);
- prompts = gptRequestBody.get("messages").getAsJsonArray();
+ gptRequestBody = getGson().fromJson(patchSetReviewer.getChatGptClient().getRequestBody(), JsonObject.class);
return reviewInputCaptor;
}
diff --git a/src/test/resources/__files/chatGptRunStepsResponse.json b/src/test/resources/__files/chatGptRunStepsResponse.json
new file mode 100644
index 0000000..30fc5e8
--- /dev/null
+++ b/src/test/resources/__files/chatGptRunStepsResponse.json
@@ -0,0 +1,27 @@
+{
+ "object": "list",
+ "data": [
+ {
+ "id": "step_TEST_STEP_ID",
+ "object": "thread.run.step",
+ "created_at": 1714983398.0,
+ "run_id": "run_TEST_RUN_ID",
+ "assistant_id": "asst_TEST_ASSISTANT_ID",
+ "thread_id": "thread_TEST_THREAD_ID",
+ "type": "tool_calls",
+ "step_details": {
+ "type": "tool_calls",
+ "tool_calls": [
+ {
+ "id": "call_8xIUWZjqjw4UKJOY58jmINKX",
+ "type": "function",
+ "function": {
+ "name": "format_replies",
+ "arguments": "{\n \"replies\": [\n {\n \"reply\": \"The change in the `rsplit` function call from `rsplit('.', 1)` to `rsplit('.', 2)` might lead to a `ValueError` if the `module_name` does not contain any dots. This change assumes that there is always at least one dot in the `module_name`. Ensure that the module naming convention enforces this or add error handling for the case where `module_name` does not contain a dot.\",\n \n\"score\n\": -1,\n \n\"relevance\n\": 0.9,\n \n\"repeated\n\": false,\n \n\"conflicting\n\": false,\n \n\"filename\n\": \n\"test_file_1.py\n\",\n \n\"lineNumber\n\": 18,\n \n\"codeSnippet\n\": \n\"module_name, class_name = module_name.rsplit('.', 2)\n\"\n }\n ],\n \n\"changeId\n\": \n\"myProject~myBranchName~myChangeId\n\"\n }"
+ }
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/src/test/resources/__files/gerritFormattedPatch.txt b/src/test/resources/__files/gerritFormattedPatch.txt
new file mode 100644
index 0000000..b7875ed
--- /dev/null
+++ b/src/test/resources/__files/gerritFormattedPatch.txt
@@ -0,0 +1,12 @@
+---
+
+diff --git a/test_file_1.py b/test_file_1.py
+index 1ece72a..a14c303 100644
+--- a/test_file_1.py
++++ b/test_file_1.py
+@@ -18,7 +18,7 @@
+"""
+ if not class_name:
+- module_name, class_name = module_name.rsplit('.', 1)
++ module_name, class_name = module_name.rsplit('.', 2)
+ loaded_module = importclass(module_name, fromlist=[class_name])
