GrepServlet: add JSON content search endpoint Add a +grep endpoint that searches blob contents at one revision using JGit TreeWalk and ObjectReader. This starts the git grep part of the Gitiles search issue without adding an index or UI. The endpoint is JSON-only for now and keeps the search bounded by skipping binary or large blobs and stopping after a fixed number of matches. Issue: 376381593 Change-Id: I351e81bb378d2a759c6b5c6d686199c598c6e886
diff --git a/Documentation/api-reference.md b/Documentation/api-reference.md index c95d9c4..8d736cd 100644 --- a/Documentation/api-reference.md +++ b/Documentation/api-reference.md
@@ -39,6 +39,20 @@ The final page will have no `next` key. Every page except for the first will have a `previous` cursor to page backwards. +#### **`+grep`** +`https://gerrit.googlesource.com/a/gitiles/+grep/refs/heads/master?s=text&format=JSON` + +Searches file contents at a revision. + +##### Search parameters +* `s=<text>` lists matches containing the case-sensitive literal substring + `<text>`. No search-syntax escaping is required. When constructing the URL + directly, the value must use standard URL encoding. + +A file or directory path may be supplied after the revision to limit the search. +The search is a case-sensitive literal substring search. Binary files and +blobs larger than 1 MiB are skipped. Results are limited to 1000 matches. + #### **`+show`** `https://gerrit.googlesource.com/a/gitiles/+show/refs/heads/master/?format=JSON` @@ -76,4 +90,4 @@ #### **`+diff`** `https://gerrit.googlesource.com/a/gitiles/+diff/refs/heads/master/?from=master~1&to=master` -Compute the diff between two commits. \ No newline at end of file +Compute the diff between two commits.
diff --git a/java/com/google/gitiles/GitilesFilter.java b/java/com/google/gitiles/GitilesFilter.java index 41c6ea4..3f35e9f 100644 --- a/java/com/google/gitiles/GitilesFilter.java +++ b/java/com/google/gitiles/GitilesFilter.java
@@ -263,6 +263,8 @@ return new DiffServlet(accessFactory, renderer, linkifier()); case LOG: return new LogServlet(accessFactory, renderer, linkifier()); + case GREP: + return new GrepServlet(accessFactory); case DESCRIBE: return new DescribeServlet(accessFactory); case ARCHIVE:
diff --git a/java/com/google/gitiles/GitilesView.java b/java/com/google/gitiles/GitilesView.java index 86b39bc..bf53759 100644 --- a/java/com/google/gitiles/GitilesView.java +++ b/java/com/google/gitiles/GitilesView.java
@@ -64,6 +64,7 @@ SHOW, DIFF, LOG, + GREP, DESCRIBE, ARCHIVE, BLAME, @@ -119,6 +120,7 @@ case ROOTED_DOC: case ARCHIVE: case BLAME: + case GREP: case SHOW: path = other.path; // $FALL-THROUGH$ @@ -200,6 +202,7 @@ case DIFF: case DOC: case LOG: + case GREP: case PATH: case REVISION: case ROOTED_DOC: @@ -259,6 +262,7 @@ case DESCRIBE: case REFS: case LOG: + case GREP: case DOC: case ROOTED_DOC: break; @@ -353,6 +357,9 @@ case LOG: checkLog(); break; + case GREP: + checkGrep(); + break; case ARCHIVE: checkArchive(); break; @@ -419,6 +426,10 @@ checkRepositoryIndex(); } + private void checkGrep() { + checkRevision(); + } + private void checkPath() { checkView(path != null, "missing path on %s view", type); checkRevision(); @@ -476,6 +487,10 @@ return new Builder(Type.LOG); } + public static Builder grep() { + return new Builder(Type.GREP); + } + public static Builder archive() { return new Builder(Type.ARCHIVE); } @@ -699,6 +714,12 @@ } } break; + case GREP: + url.append(repositoryName).append("/+grep/").append(revision.getName()); + if (path != null) { + url.append('/').append(path); + } + break; case BLAME: url.append(repositoryName) .append("/+blame/")
diff --git a/java/com/google/gitiles/GrepServlet.java b/java/com/google/gitiles/GrepServlet.java new file mode 100644 index 0000000..bae8139 --- /dev/null +++ b/java/com/google/gitiles/GrepServlet.java
@@ -0,0 +1,208 @@ +// Copyright 2026 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gitiles; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.gitiles.GitilesRequestFailureException.FailureReason; +import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.util.List; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.http.server.ServletUtils; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.util.RawParseUtils; + +/** Serves file-content search results for a repository tree. */ +public class GrepServlet extends BaseServlet { + private static final long serialVersionUID = 1L; + + private static final String SUBSTRING_PARAM = "s"; + @VisibleForTesting static final int MAX_MATCHES = 1000; + private static final int MAX_BLOB_SIZE = 1 << 20; // 1 MB + + protected GrepServlet(GitilesAccess.Factory accessFactory) { + super(null, accessFactory); + } + + @Override + protected void doGetJson(HttpServletRequest req, HttpServletResponse res) throws IOException { + GitilesView view = ViewFilter.getView(req); + String substring = Iterables.getFirst(view.getParameters().get(SUBSTRING_PARAM), null); + if (Strings.isNullOrEmpty(substring)) { + throw new GitilesRequestFailureException(FailureReason.INCORRECT_PARAMETER) + .withPublicErrorMessage("missing s parameter"); + } + + GrepResult result = grep(ServletUtils.getRepository(req), view, substring); + renderJson(req, res, result, new TypeToken<GrepResult>() {}.getType()); + } + + private static GrepResult grep(Repository repo, GitilesView view, String substring) + throws IOException { + List<Match> matches = Lists.newArrayList(); + + try (RevWalk rw = new RevWalk(repo)) { + RevTree root = getRoot(view, rw); + String path = Strings.nullToEmpty(view.getPathPart()); + if (path.isEmpty()) { + grepTree(rw.getObjectReader(), root, "", substring, matches); + } else { + grepPath(rw.getObjectReader(), root, path, substring, matches); + } + } + + return new GrepResult(matches); + } + + private static void grepPath( + ObjectReader reader, RevTree root, String path, String substring, List<Match> matches) + throws IOException { + try (TreeWalk tw = TreeWalk.forPath(reader, path, root)) { + if (tw == null) { + throw new GitilesRequestFailureException(FailureReason.OBJECT_NOT_FOUND); + } + + FileMode mode = tw.getFileMode(0); + ObjectId id = tw.getObjectId(0); + if (mode.getObjectType() == Constants.OBJ_BLOB) { + grepBlob(reader, id, path, substring, matches); + return; + } + if (mode.getObjectType() != Constants.OBJ_TREE) { + return; + } + + try (TreeWalk subtree = new TreeWalk(reader)) { + subtree.addTree(id); + subtree.setRecursive(true); + grepTree(subtree, path + "/", substring, matches); + } + } + } + + private static void grepTree( + ObjectReader reader, RevTree root, String pathPrefix, String substring, List<Match> matches) + throws IOException { + try (TreeWalk tw = new TreeWalk(reader)) { + tw.addTree(root); + tw.setRecursive(true); + grepTree(tw, pathPrefix, substring, matches); + } + } + + private static void grepTree( + TreeWalk tw, String pathPrefix, String substring, List<Match> matches) throws IOException { + while (tw.next()) { + if (matches.size() >= MAX_MATCHES) { + return; + } + if (tw.getFileMode(0).getObjectType() != Constants.OBJ_BLOB) { + continue; + } + grepBlob( + tw.getObjectReader(), + tw.getObjectId(0), + pathPrefix + tw.getPathString(), + substring, + matches); + if (matches.size() >= MAX_MATCHES) { + return; + } + } + } + + private static void grepBlob( + ObjectReader reader, ObjectId id, String path, String substring, List<Match> matches) + throws IOException { + ObjectLoader loader = reader.open(id, Constants.OBJ_BLOB); + if (loader.getSize() > MAX_BLOB_SIZE) { + return; + } + + byte[] raw; + try { + raw = loader.getCachedBytes(MAX_BLOB_SIZE); + } catch (LargeObjectException e) { + return; + } + if (RawText.isBinary(raw)) { + return; + } + + String[] lines = RawParseUtils.decode(raw).split("\n", -1); + for (int i = 0; i < lines.length; i++) { + if (matches.size() >= MAX_MATCHES) { + return; + } + String line = + lines[i].endsWith("\r") ? lines[i].substring(0, lines[i].length() - 1) : lines[i]; + if (line.contains(substring)) { + matches.add(new Match(path, i + 1, line)); + if (matches.size() >= MAX_MATCHES) { + return; + } + } + } + } + + private static RevTree getRoot(GitilesView view, RevWalk rw) throws IOException { + RevObject obj = rw.peel(rw.parseAny(view.getRevision().getId())); + switch (obj.getType()) { + case Constants.OBJ_COMMIT: + return ((RevCommit) obj).getTree(); + case Constants.OBJ_TREE: + return (RevTree) obj; + default: + throw new GitilesRequestFailureException(FailureReason.INCORRECT_OBJECT_TYPE) + .withPublicErrorMessage("The specified object is not a tree-ish."); + } + } + + static class GrepResult { + List<Match> matches; + + GrepResult(List<Match> matches) { + this.matches = matches; + } + } + + static class Match { + String path; + int lineNumber; + String line; + + Match(String path, int lineNumber, String line) { + this.path = path; + this.lineNumber = lineNumber; + this.line = line; + } + } +}
diff --git a/java/com/google/gitiles/ViewFilter.java b/java/com/google/gitiles/ViewFilter.java index 2cbcdff..a7e6a15 100644 --- a/java/com/google/gitiles/ViewFilter.java +++ b/java/com/google/gitiles/ViewFilter.java
@@ -46,6 +46,7 @@ private static final String CMD_DESCRIBE = "+describe"; private static final String CMD_DIFF = "+diff"; private static final String CMD_LOG = "+log"; + private static final String CMD_GREP = "+grep"; private static final String CMD_REFS = "+refs"; private static final String CMD_SHOW = "+show"; private static final String CMD_DOC = "+doc"; @@ -163,6 +164,8 @@ return parseDiffCommand(req, repoName, path); } else if (command.equals(CMD_LOG)) { return parseLogCommand(req, repoName, path); + } else if (command.equals(CMD_GREP)) { + return parseGrepCommand(req, repoName, path); } else if (command.equals(CMD_REFS)) { return parseRefsCommand(repoName, path); } else if (command.equals(CMD_SHOW)) { @@ -285,6 +288,21 @@ .setPathPart(result.getPath()); } + private @Nullable GitilesView.Builder parseGrepCommand( + HttpServletRequest req, String repoName, String path) throws IOException { + if (isEmptyOrSlash(path)) { + return null; + } + RevisionParser.Result result = parseRevision(req, path); + if (result.getOldRevision() != null) { + return null; + } + return GitilesView.grep() + .setRepositoryName(repoName) + .setRevision(result.getRevision()) + .setPathPart(Strings.emptyToNull(result.getPath())); + } + private GitilesView.Builder parseRefsCommand(String repoName, String path) { return GitilesView.refs().setRepositoryName(repoName).setPathPart(path); }
diff --git a/javatests/com/google/gitiles/GrepServletTest.java b/javatests/com/google/gitiles/GrepServletTest.java new file mode 100644 index 0000000..2ccd951 --- /dev/null +++ b/javatests/com/google/gitiles/GrepServletTest.java
@@ -0,0 +1,93 @@ +// Copyright 2026 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gitiles; + +import static com.google.common.truth.Truth.assertThat; +import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; +import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; + +import java.util.List; +import org.junit.Test; + +public class GrepServletTest extends ServletTest { + @Test + public void grepJsonSearchesRepositoryRoot() throws Exception { + repo.branch("master") + .commit() + .add("dir/a.txt", "alpha\nneedle here\n") + .add("dir/b.txt", "needle too\n") + .add("dir/c.txt", "no match\n") + .create(); + + GrepResult result = buildJson(GrepResult.class, "/repo/+grep/master", "s=needle"); + + assertThat(result.matches).hasSize(2); + assertThat(result.matches.get(0).path).isEqualTo("dir/a.txt"); + assertThat(result.matches.get(0).lineNumber).isEqualTo(2); + assertThat(result.matches.get(0).line).isEqualTo("needle here"); + assertThat(result.matches.get(1).path).isEqualTo("dir/b.txt"); + } + + @Test + public void grepJsonCanSearchDirectoryPath() throws Exception { + repo.branch("master") + .commit() + .add("src/a.txt", "needle\n") + .add("test/a.txt", "needle\n") + .create(); + + GrepResult result = buildJson(GrepResult.class, "/repo/+grep/master/src", "s=needle"); + + assertThat(result.matches).hasSize(1); + assertThat(result.matches.get(0).path).isEqualTo("src/a.txt"); + } + + @Test + public void grepJsonDoesNotSearchNonExistingPathPrefix() throws Exception { + repo.branch("master").commit().add("src/a.txt", "needle\n").create(); + + buildResponse("/repo/+grep/master/sr", "s=needle&format=JSON", SC_NOT_FOUND); + } + + @Test + public void grepJsonLimitsMatches() throws Exception { + var commit = repo.branch("master").commit(); + for (int i = 0; i < GrepServlet.MAX_MATCHES + 1; i++) { + commit.add(String.format("file%04d.txt", i), "needle\n"); + } + commit.create(); + + GrepResult result = buildJson(GrepResult.class, "/repo/+grep/master", "s=needle"); + + assertThat(result.matches).hasSize(GrepServlet.MAX_MATCHES); + } + + @Test + public void grepJsonRequiresSubstring() throws Exception { + repo.branch("master").commit().add("foo", "contents").create(); + + buildResponse("/repo/+grep/master", "format=JSON", SC_BAD_REQUEST); + } + + private static class GrepResult { + List<Match> matches; + } + + private static class Match { + String path; + int lineNumber; + String line; + } +}