REST/SSH API to extract project contributors
Initial implementation of the project contributors list extracting
via REST and SSH.
Differently from the standard Gerrit API, list of entries is returned
as multi-line JSON instead of a unique collection in a single line.
The rationale is on the potential size of the data and the underlying
processing system which is based on splitting, shuffling and sorting.
Change-Id: I96b846d732f81cc35afb6520b4c3130f5cb8b9bd
diff --git a/BUCK b/BUCK
index 2868155..7887a62 100644
--- a/BUCK
+++ b/BUCK
@@ -1,4 +1,5 @@
include_defs('//bucklets/gerrit_plugin.bucklet')
+include_defs('//lib/maven.defs')
gerrit_plugin(
name = 'analytics',
@@ -6,10 +7,13 @@
resources = glob(['src/main/resources/**/*']),
manifest_entries = [
'Gerrit-PluginName: analytics',
+ 'Gerrit-Module: com.googlesource.gerrit.plugins.analytics.Module',
+ 'Gerrit-SshModule: com.googlesource.gerrit.plugins.analytics.SshModule',
'Implementation-Title: Analytics plugin',
'Implementation-URL: https://gerrit-review.googlesource.com/#/admin/projects/plugins/analytics',
],
deps = [
+ ':gitective-core',
],
)
@@ -21,3 +25,10 @@
':analytics__plugin',
],
)
+
+maven_jar(
+ name = 'gitective-core',
+ id = 'io.fabric8:gitective-core:0.9.18-1',
+ repository = MAVEN_LOCAL,
+ license = 'Apache2.0',
+)
diff --git a/README.md b/README.md
index 7cf1812..d79db9c 100644
--- a/README.md
+++ b/README.md
@@ -3,4 +3,72 @@
Extract commit and review data from Gerrit projects and expose aggregated metrics
over REST and SSH API.
+## How to build
+
+Clone the analytics plugin into an existing Gerrit source tree under /plugins/analytics
+and then execute buck build.
+
+Example:
+
+```
+ $ git clone https://gerrit.googlesource.com/gerrit
+ $ git clone https://gerrit.googlesource.com/plugins/analytics gerrit/plugins/analytics
+ $ cd gerrit
+ $ buck build plugins/analytics
+```
+
+## How to install
+
+Copy the analytics.jar generated under /buck-out/gen/plugins/analytics/analytics.jar
+onto Gerrit's /plugins directory.
+
+## How to configure
+
+Nothing to configure, it just works.
+
+## How to use
+
+Adds new REST API and SSH commands to allow the extraction of repository statistics
+from Gerrit repositories and changes.
+
+## API
+
+All the API share the same syntax and behaviour. Differently from the standard Gerrit REST
+API, the JSON collections are returned as individual lines and streamed over the socket
+I/O. The choice is driven by the fact that the typical consumer of these API is a BigData
+batch process, typically external to Gerrit and hosted on a separate computing cluster.
+
+A large volume of data can be potentially generated: splitting the output file into separate
+lines helps the BigData processing in the splitting, shuffling and sorting phase.
+
+### Contributors
+
+Extract a unordered list of project contributors statistics, including the commits data
+relevant for statistics purposes, such as timestamp and merge flag.
+
+*REST*
+
+/projects/{project-name}/analytics~contributors
+
+*SSH*
+
+analytics contributors {project-name}
+
+REST Example:
+
+```
+ $ curl http://gerrit.mycompany.com/project/myproyject/analytics~contributors
+
+ {"name":"John Doe","email":"john.doe@mycompany.com","num_commits":1,"commits":[{"sha1":"6a1f73738071e299f600017d99f7252d41b96b4b","date":"Apr 28, 2011 5:13:14 AM","merge":false}]}
+ {"name":"Matt Smith","email":"matt.smith@mycompany.com","num_commits":1,"commits":[{"sha1":"54527e7e3086758a23e3b069f183db6415aca304","date":"Sep 8, 2015 3:11:23 AM","merge":true}]}
+```
+
+SSH Example:
+
+```
+ $ ssh -p 29418 admin@gerrit.mycompany.com analytics contributors
+
+ {"name":"John Doe","email":"john.doe@mycompany.com","num_commits":1,"commits":[{"sha1":"6a1f73738071e299f600017d99f7252d41b96b4b","date":"Apr 28, 2011 5:13:14 AM","merge":false}]}
+ {"name":"Matt Smith","email":"matt.smith@mycompany.com","num_commits":1,"commits":[{"sha1":"54527e7e3086758a23e3b069f183db6415aca304","date":"Sep 8, 2015 3:11:23 AM","merge":true}]}
+```
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/CommitInfo.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/CommitInfo.java
new file mode 100644
index 0000000..807e516
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/CommitInfo.java
@@ -0,0 +1,30 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+import java.util.Date;
+
+public class CommitInfo {
+ public final String sha1;
+ public final Date date;
+ public final boolean merge;
+
+ public CommitInfo(String sha1, Date date, boolean merge) {
+ super();
+ this.sha1 = sha1;
+ this.date = date;
+ this.merge = merge;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/ContributorsCommand.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/ContributorsCommand.java
new file mode 100644
index 0000000..0105ea5
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/ContributorsCommand.java
@@ -0,0 +1,60 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
+import com.google.gerrit.server.project.ProjectResource;
+import com.google.gerrit.server.project.ProjectsCollection;
+import com.google.gerrit.sshd.CommandMetaData;
+import com.google.gerrit.sshd.SshCommand;
+import com.google.inject.Inject;
+
+import org.kohsuke.args4j.Argument;
+
+import java.io.IOException;
+
+@CommandMetaData(name = "contributors", description = "Extracts the list of contributors to a project")
+public class ContributorsCommand extends SshCommand {
+ private final ProjectsCollection projects;
+ private final ContributorsResource contributors;
+ private final GsonFormatter gsonFmt;
+
+ @Inject
+ public ContributorsCommand(ProjectsCollection projects,
+ ContributorsResource contributors, GsonFormatter gsonFmt) {
+ this.projects = projects;
+ this.contributors = contributors;
+ this.gsonFmt = gsonFmt;
+ }
+
+ @Argument(usage = "project name", metaVar = "PROJECT", required = true)
+ void setProject(String project) throws IllegalArgumentException {
+ try {
+ this.projectRes = projects.parse(project);
+ } catch (UnprocessableEntityException e) {
+ throw new IllegalArgumentException(e.getLocalizedMessage(), e);
+ } catch (IOException e) {
+ throw new IllegalArgumentException(
+ "I/O Error while trying to access project " + project, e);
+ }
+ }
+
+ private ProjectResource projectRes;
+
+ @Override
+ protected void run() throws UnloggedFailure, Failure, Exception {
+ gsonFmt.format(contributors.getStream(projectRes), stdout);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/ContributorsResource.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/ContributorsResource.java
new file mode 100644
index 0000000..d9963de
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/ContributorsResource.java
@@ -0,0 +1,72 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+import com.google.gerrit.extensions.restapi.BinaryResult;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.RestReadView;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.project.ProjectResource;
+import com.google.inject.Inject;
+
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.Repository;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.stream.Stream;
+
+class ContributorsResource implements RestReadView<ProjectResource> {
+ private final GitRepositoryManager repoManager;
+ private final UserSummaryExport userSummary;
+ private final GsonFormatter gsonFmt;
+
+ class JsonStreamedResult<T> extends BinaryResult {
+ private final Stream<T> committers;
+
+ public JsonStreamedResult(Stream<T> committers) {
+ this.committers = committers;
+ }
+
+ @Override
+ public void writeTo(OutputStream os) throws IOException {
+ try (PrintWriter sout = new PrintWriter(os)) {
+ gsonFmt.format(committers, sout);
+ }
+ }
+ }
+
+ @Inject
+ public ContributorsResource(GitRepositoryManager repoManager,
+ UserSummaryExport userSummary, GsonFormatter gsonFmt) {
+ this.repoManager = repoManager;
+ this.userSummary = userSummary;
+ this.gsonFmt = gsonFmt;
+ }
+
+ @Override
+ public Response<BinaryResult> apply(ProjectResource projectRes)
+ throws RepositoryNotFoundException, IOException {
+ return Response.ok(new JsonStreamedResult<>(getStream(projectRes)));
+ }
+
+ public Stream<UserActivitySummary> getStream(ProjectResource projectRes)
+ throws RepositoryNotFoundException, IOException {
+ try (Repository repo = repoManager.openRepository(projectRes.getNameKey())) {
+ return userSummary.getCommittersStream(repo);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/GsonFormatter.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/GsonFormatter.java
new file mode 100644
index 0000000..eecd33b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/GsonFormatter.java
@@ -0,0 +1,43 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+import com.google.gerrit.server.OutputFormat;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import java.io.PrintWriter;
+import java.util.stream.Stream;
+
+@Singleton
+public class GsonFormatter {
+ private GsonBuilder gsonBuilder;
+
+ @Inject
+ public GsonFormatter() {
+ gsonBuilder = OutputFormat.JSON_COMPACT.newGsonBuilder();
+ }
+
+ public <T> void format(Stream<T> values, PrintWriter out) {
+ final Gson gson = gsonBuilder.create();
+
+ values.sequential().forEach((T value) -> {
+ gson.toJson(value, out);
+ out.println();
+ });
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/Module.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/Module.java
new file mode 100644
index 0000000..3d2f6eb
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/Module.java
@@ -0,0 +1,34 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+import static com.google.gerrit.server.project.ProjectResource.PROJECT_KIND;
+
+import com.google.gerrit.extensions.restapi.RestApiModule;
+import com.google.inject.AbstractModule;
+
+public class Module extends AbstractModule {
+
+ @Override
+ protected void configure() {
+
+ install(new RestApiModule() {
+ @Override
+ protected void configure() {
+ get(PROJECT_KIND, "contributors").to(ContributorsResource.class);
+ }
+ });
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/SshModule.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/SshModule.java
new file mode 100644
index 0000000..7f015f9
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/SshModule.java
@@ -0,0 +1,25 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+import com.google.gerrit.sshd.PluginCommandModule;
+
+public class SshModule extends PluginCommandModule {
+
+ @Override
+ protected void configureCommands() {
+ command(ContributorsCommand.class);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/UserActivitySummary.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/UserActivitySummary.java
new file mode 100644
index 0000000..09621cf
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/UserActivitySummary.java
@@ -0,0 +1,57 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+
+import org.eclipse.jgit.lib.ObjectId;
+// import org.gitective.core.stat.UserCommitActivity;
+import org.gitective.core.stat.UserCommitActivity;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class UserActivitySummary {
+ public final String name;
+ public final String email;
+ public final int numCommits;
+ public final List<CommitInfo> commits;
+
+
+ public UserActivitySummary(String name, String email, int numCommits,
+ List<CommitInfo> commits) {
+ this.name = name;
+ this.email = email;
+ this.numCommits = numCommits;
+ this.commits = commits;
+ }
+
+ public static UserActivitySummary fromUserActivity(UserCommitActivity uca) {
+ return new UserActivitySummary(uca.getName(), uca.getEmail(),
+ uca.getCount(), getCommits(uca.getIds(), uca.getTimes(),
+ uca.getMerges()));
+ }
+
+ private static List<CommitInfo> getCommits(ObjectId[] ids, long[] times,
+ boolean[] merges) {
+ List<CommitInfo> commits = new ArrayList<>(ids.length);
+
+ for (int i = 0; i < ids.length; i++) {
+ commits.add(new CommitInfo(ids[i].name(), new Date(times[i]), merges[i]));
+ }
+
+ return commits;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/analytics/UserSummaryExport.java b/src/main/java/com/googlesource/gerrit/plugins/analytics/UserSummaryExport.java
new file mode 100644
index 0000000..953dc3a
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/analytics/UserSummaryExport.java
@@ -0,0 +1,42 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics;
+
+import com.google.inject.Singleton;
+
+import org.eclipse.jgit.lib.Repository;
+import org.gitective.core.CommitFinder;
+import org.gitective.core.stat.AuthorHistogramFilter;
+import org.gitective.core.stat.CommitHistogram;
+import org.gitective.core.stat.CommitHistogramFilter;
+import org.gitective.core.stat.UserCommitActivity;
+
+import java.util.stream.Stream;
+
+@Singleton
+public class UserSummaryExport {
+
+ public Stream<UserActivitySummary> getCommittersStream(Repository repo) {
+ CommitFinder finder = new CommitFinder(repo);
+ CommitHistogramFilter filter = new AuthorHistogramFilter();
+ finder.setFilter(filter).find();
+ CommitHistogram histogram = filter.getHistogram();
+ UserCommitActivity[] authorActivity = histogram.getUserActivity();
+
+ return Stream.of(authorActivity)
+ .parallel()
+ .map(UserActivitySummary::fromUserActivity);
+ }
+}
