Add Issues info for enabling ETL aggregation
Extract issues_codes and issues_links arrays from
the commit message description and add to Statistics.
See:
https://gerrit-review.googlesource.com/Documentation/config-gerrit.html#commentlink
Change-Id: Ibc0d3a4b26297281f56ccecd81c9c953dc335472
Jira-Id: GERICS-628
diff --git a/README.md b/README.md
index 1146e93..29bc565 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,10 @@
commits data relevant for statistics purposes, such as number of involved files, and optionally also the list of belonging branches,
number of added/deleted lines, timestamp and merge flag.
+Optionally, extract information on issues using the [commentLink](https://gerrit-review.googlesource.com/Documentation/config-gerrit.html#commentlink)
+Gerrit configuration and enrich the statistics with the issue-ids and links obtained from
+the commit message.
+
*REST*
@@ -66,6 +70,7 @@
- --until -e Ending timestamp (excluded) to consider
- --aggregate -granularity -g one of email, email_year, email_month, email_day, email_hour defaulting to aggregation by email
- --extract-branches -r enables branches extraction for each commit
+- --extract-issues -i enables the extraction of issues from commentLink
NOTE: Timestamp format is consistent with Gerrit's query syntax, see /Documentation/user-search.html for details.
@@ -80,8 +85,8 @@
SSH Example:
```
- $ ssh -p 29418 admin@gerrit.mycompany.com analytics contributors myproject --since 2017-08-01 --until 2017-12-31
- {"name":"John Doe","email":"john.doe@mycompany.com","num_commits":1, "num_files":4,"added_lines":9,"deleted_lines":1, "commits":[{"sha1":"6a1f73738071e299f600017d99f7252d41b96b4b","date":"Apr 28, 2011 5:13:14 AM","merge":false}]}
- {"name":"Matt Smith","email":"matt.smith@mycompany.com","num_commits":1, "num_files":1,"added_lines":90,"deleted_lines":10,"commits":[{"sha1":"54527e7e3086758a23e3b069f183db6415aca304","date":"Sep 8, 2015 3:11:23 AM","merge":true}],"branches":["master","branch1"]}
+ $ ssh -p 29418 admin@gerrit.mycompany.com analytics contributors myproject --since 2017-08-01 --until 2017-12-31 --extract-issues
+ {"name":"John Doe","email":"john.doe@mycompany.com","num_commits":1, "num_files":4,"added_lines":9,"deleted_lines":1, "commits":[{"sha1":"6a1f73738071e299f600017d99f7252d41b96b4b","date":"Apr 28, 2011 5:13:14 AM","merge":false}], "issues_codes":["PRJ-001"],"issues_links":["https://jira.company.org/PRJ-001"]}
+ {"name":"Matt Smith","email":"matt.smith@mycompany.com","num_commits":1, "num_files":1,"added_lines":90,"deleted_lines":10,"commits":[{"sha1":"54527e7e3086758a23e3b069f183db6415aca304","date":"Sep 8, 2015 3:11:23 AM","merge":true}],"branches":["master","branch1"],"issues_codes":["PRJ-002","PRJ-003"],"issues_links":["https://jira.company.org/PRJ-002","https://jira.company.org/PRJ-003"]}
```
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/Contributors.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/Contributors.scala
index f24d996..3c1253a 100644
--- a/src/main/scala/com/googlesource/gerrit/plugins/analytics/Contributors.scala
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/Contributors.scala
@@ -14,12 +14,11 @@
package com.googlesource.gerrit.plugins.analytics
+import com.google.gerrit.extensions.api.projects.CommentLinkInfo
import com.google.gerrit.extensions.restapi.{BadRequestException, Response, RestReadView}
import com.google.gerrit.server.git.GitRepositoryManager
-import com.google.gerrit.server.project.{ProjectResource, ProjectsCollection}
+import com.google.gerrit.server.project.{ProjectCache, ProjectResource, ProjectsCollection}
import com.google.gerrit.sshd.{CommandMetaData, SshCommand}
-import com.google.gson.TypeAdapter
-import com.google.gson.stream.{JsonReader, JsonWriter}
import com.google.inject.Inject
import com.googlesource.gerrit.plugins.analytics.common.DateConversions._
import com.googlesource.gerrit.plugins.analytics.common._
@@ -70,9 +69,13 @@
}
}
+ @ArgOption(name = "--extract-issues", aliases = Array("-i"),
+ usage = "Extract a list of issues and links using the Gerrit's commentLink configuration")
+ private var extractIssues: Boolean = false
+
override protected def run =
gsonFmt.format(executor.get(projectRes, beginDate, endDate,
- granularity.getOrElse(AggregationStrategy.EMAIL), extractBranches), stdout)
+ granularity.getOrElse(AggregationStrategy.EMAIL), extractBranches, extractIssues), stdout)
}
@@ -118,22 +121,34 @@
usage = "Do extra parsing to extract a list of all branches for each line")
private var extractBranches: Boolean = false
+ @ArgOption(name = "--extract-issues", aliases = Array("-i"),
+ usage = "Extract a list of issues and links using the Gerrit's commentLink configuration")
+ private var extractIssues: Boolean = false
+
override def apply(projectRes: ProjectResource) =
Response.ok(
new GsonStreamedResult[UserActivitySummary](gson,
executor.get(projectRes, beginDate, endDate,
- granularity.getOrElse(AggregationStrategy.EMAIL), extractBranches)))
+ granularity.getOrElse(AggregationStrategy.EMAIL), extractBranches, extractIssues)))
}
class ContributorsService @Inject()(repoManager: GitRepositoryManager,
+ projectCache:ProjectCache,
histogram: UserActivityHistogram,
gsonFmt: GsonFormatter) {
+ import RichBoolean._
+ import scala.collection.JavaConverters._
+
def get(projectRes: ProjectResource, startDate: Option[Long], stopDate: Option[Long],
- aggregationStrategy: AggregationStrategy, extractBranches: Boolean)
+ aggregationStrategy: AggregationStrategy, extractBranches: Boolean, extractIssues: Boolean)
: TraversableOnce[UserActivitySummary] = {
+ val nameKey = projectRes.getNameKey
+ val commentLinks: List[CommentLinkInfo] = extractIssues.option {
+ projectCache.get(nameKey).getCommentLinks.asScala
+ }.toList.flatten
+
ManagedResource.use(repoManager.openRepository(projectRes.getNameKey)) { repo =>
- val stats = new Statistics(repo)
- import RichBoolean._
+ val stats = new Statistics(repo, commentLinks.asJava)
val commitsBranchesOptionalEnricher = extractBranches.option(
new CommitsBranches(repo, startDate, stopDate)
)
@@ -148,6 +163,8 @@
case class CommitInfo(sha1: String, date: Long, merge: Boolean, files: java.util.Set[String])
+case class IssueInfo(code: String, link: String)
+
case class UserActivitySummary(year: Integer,
month: Integer,
day: Integer,
@@ -161,6 +178,8 @@
deletedLines: Integer,
commits: Array[CommitInfo],
branches: Array[String],
+ issuesCodes: Array[String],
+ issuesLinks: Array[String],
lastCommitDate: Long,
isMerge: Boolean
)
@@ -183,7 +202,9 @@
UserActivitySummary(
year, month, day, hour, uca.getName, uca.getEmail, uca.getCount,
stat.numFiles, stat.numDistinctFiles, stat.addedLines, stat.deletedLines,
- stat.commits.toArray, branches.toArray, uca.getLatest, stat.isForMergeCommits
+ stat.commits.toArray, branches.toArray, stat.issues.map(_.code)
+ .toArray, stat.issues.map(_.link).toArray, uca.getLatest, stat
+ .isForMergeCommits
)
}
case _ => throw new Exception(s"invalid key format found ${uca.key}")
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/common/CommitsStatistics.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/common/CommitsStatistics.scala
index e5aae6c..2dc7515 100644
--- a/src/main/scala/com/googlesource/gerrit/plugins/analytics/common/CommitsStatistics.scala
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/common/CommitsStatistics.scala
@@ -14,15 +14,18 @@
package com.googlesource.gerrit.plugins.analytics.common
-import com.googlesource.gerrit.plugins.analytics.CommitInfo
+import com.google.gerrit.extensions.api.projects.CommentLinkInfo
+import com.googlesource.gerrit.plugins.analytics.{CommitInfo, IssueInfo}
import com.googlesource.gerrit.plugins.analytics.common.ManagedResource.use
import org.eclipse.jgit.diff.{DiffFormatter, RawTextComparator}
import org.eclipse.jgit.lib.{ObjectId, Repository}
-import org.eclipse.jgit.revwalk.{RevCommit, RevWalk}
+import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.treewalk.{CanonicalTreeParser, EmptyTreeIterator}
import org.eclipse.jgit.util.io.DisabledOutputStream
+import org.slf4j.LoggerFactory
import scala.collection.JavaConversions._
+import scala.util.matching.Regex
/**
* Collects overall stats on a series of commits and provides some basic info on the included commits
@@ -38,7 +41,8 @@
addedLines: Int,
deletedLines: Int,
isForMergeCommits: Boolean,
- commits: List[CommitInfo]
+ commits: List[CommitInfo],
+ issues: List[IssueInfo] = Nil
) {
require(commits.forall(_.merge == isForMergeCommits), s"Creating a stats object with isMergeCommit = $isForMergeCommits but containing commits of different type")
@@ -62,17 +66,24 @@
this.copy(
addedLines = this.addedLines + that.addedLines,
deletedLines = this.deletedLines + that.deletedLines,
- commits = this.commits ++ that.commits
+ commits = this.commits ++ that.commits,
+ issues = this.issues ++ that.issues
)
}
}
object CommitsStatistics {
- val Empty = CommitsStatistics(0, 0, false, List.empty)
+ val Empty = CommitsStatistics(0, 0, false, List[CommitInfo](), List[IssueInfo]())
val EmptyMerge = Empty.copy(isForMergeCommits = true)
}
-class Statistics(repo: Repository) {
+class Statistics(repo: Repository, commentInfoList: java.util.List[CommentLinkInfo] = Nil) {
+
+ val log = LoggerFactory.getLogger(classOf[Statistics])
+ val replacers = commentInfoList.map(info =>
+ Replacer(
+ info.`match`.r,
+ Option(info.link).getOrElse(info.html)))
/**
* Returns up to two different CommitsStatistics object grouping the stats into merge and non-merge commits
@@ -98,6 +109,7 @@
use(new RevWalk(repo)) { rw =>
val reader = repo.newObjectReader()
val commit = rw.parseCommit(objectId)
+ val commitMessage = commit.getFullMessage
val oldTree = {
// protects against initial commit
@@ -126,8 +138,21 @@
val commitInfo = CommitInfo(objectId.getName, commit.getAuthorIdent.getWhen.getTime, commit.isMerge, files)
- CommitsStatistics(lines.added, lines.deleted, commitInfo.merge, List(commitInfo))
+ CommitsStatistics(lines.added, lines.deleted, commitInfo.merge, List(commitInfo), extractIssues(commitMessage))
}
}
+ def extractIssues(commitMessage: String): List[IssueInfo] = {
+ replacers.flatMap {
+ case Replacer(pattern, replaced) =>
+ pattern.findAllIn(commitMessage)
+ .map(code => {
+ val transformed = pattern.replaceAllIn(code, replaced)
+ IssueInfo(code, transformed)
+ })
+ }.toList
+ }
+
+ case class Replacer(pattern: Regex, replaced: String)
+
}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/analytics/test/CommitStatisticsCommentLinkSpec.scala b/src/test/scala/com/googlesource/gerrit/plugins/analytics/test/CommitStatisticsCommentLinkSpec.scala
new file mode 100644
index 0000000..b5d7482
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/analytics/test/CommitStatisticsCommentLinkSpec.scala
@@ -0,0 +1,98 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.analytics.test
+
+import java.util.{Arrays, Date}
+
+import com.google.gerrit.extensions.api.projects.CommentLinkInfo
+import com.googlesource.gerrit.plugins.analytics.IssueInfo
+import com.googlesource.gerrit.plugins.analytics.common.{CommitsStatistics, Statistics}
+import org.eclipse.jgit.internal.storage.file.FileRepository
+import org.eclipse.jgit.revwalk.RevCommit
+import org.scalatest.{FlatSpec, Inside, Matchers}
+import scala.collection.JavaConverters._
+
+class CommitStatisticsCommentLinkSpec extends FlatSpec with GitTestCase with Matchers with Inside {
+
+ def createCommentLinkInfo(pattern: String, link: Option[String] = None, html: Option[String] = None) = {
+ val info = new CommentLinkInfo
+ info.`match` = pattern
+ info.link = link.getOrElse(null)
+ info.html = html.getOrElse(null)
+ info
+ }
+
+ def commit(committer: String, fileName: String, content: String, message: Option[String] = None): RevCommit = {
+ val date = new Date()
+ val person = newPersonIdent(committer, committer, date)
+ add(testRepo, fileName, content, author = person, committer = author, message = message.getOrElse("** no message **"))
+ }
+
+ class TestEnvironment(val repo: FileRepository = new FileRepository(testRepo),
+ val commentLinks: java.util.List[CommentLinkInfo] = Seq(
+ createCommentLinkInfo(pattern = "(bug\\s+#?)(\\d+)",
+ link = Some("http://bugs.example.com/show_bug.cgi?id=$2")),
+ createCommentLinkInfo(pattern = "([Bb]ug:\\s+)(\\d+)",
+ html = Some("$1<a href=\"http://trak.example.com/$2\">$2</a>"))).asJava) {
+
+ lazy val stats = new Statistics(repo, commentLinks)
+ }
+
+ it should "collect no commentslink if no matching" in new TestEnvironment {
+ val nocomments = commit("user", "file1.txt", "content1")
+
+ inside(stats.forCommits(nocomments)) {
+ case List(s: CommitsStatistics) =>
+ s.issues should have size 0
+ }
+
+ }
+ it should "collect simple bugzilla comments" in new TestEnvironment {
+ val simpleComment = commit("user", "file1.txt", "content2", message =
+ Some("this solves bug #23"))
+
+ inside(stats.forCommits(simpleComment)) {
+ case List(s: CommitsStatistics) =>
+ s.issues should have size 1
+ s.issues should contain(IssueInfo("bug #23", "http://bugs.example.com/show_bug.cgi?id=23"))
+ }
+
+ }
+ it should "collect simple track link" in new TestEnvironment {
+ val simpleTrackComment = commit("user", "file1.txt", "content3", message
+ = Some("this solves Bug: 1234"))
+
+ inside(stats.forCommits(simpleTrackComment)) {
+ case List(s: CommitsStatistics) =>
+ s.issues should have size 1
+ s.issues should contain(IssueInfo("Bug: 1234", "Bug: <a href=\"http://trak.example.com/1234\">1234</a>"))
+ }
+
+ }
+ it should "collect multiple links" in new TestEnvironment {
+ val multipleComments = commit("user", "file1.txt", "content4", message =
+ Some("this solves bug 12 and Bug: 23"))
+
+ inside(stats.forCommits(multipleComments)) {
+ case List(s: CommitsStatistics) =>
+ s.issues should contain allOf(
+ IssueInfo("bug 12", "http://bugs.example.com/show_bug.cgi?id=12"),
+ IssueInfo("Bug: 23", "Bug: <a href=\"http://trak.example.com/23\">23</a>")
+ )
+ }
+
+ }
+
+}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/analytics/test/CommitStatisticsSpec.scala b/src/test/scala/com/googlesource/gerrit/plugins/analytics/test/CommitStatisticsSpec.scala
index 0f9ba34..df4260a 100644
--- a/src/test/scala/com/googlesource/gerrit/plugins/analytics/test/CommitStatisticsSpec.scala
+++ b/src/test/scala/com/googlesource/gerrit/plugins/analytics/test/CommitStatisticsSpec.scala
@@ -152,7 +152,7 @@
nonMergeStats.addedLines should be(4)
nonMergeStats.deletedLines should be(2)
- case wrongContent => fail(s"Expected two results instad got $wrongContent")
+ case wrongContent => fail(s"Expected two results instead got $wrongContent")
}
}
