Label results and and manifest branch information Labelling the results could be used for the aggregation of projects imported from the same manifest file. Add "manifest_label" and "manifest_branch" fields to allow aggregation and filtering by information associated with the manifest. Change-Id: I9d868357852aa0386dfa10f012aebd539728a1e1
diff --git a/README.md b/README.md index a0a869f..7abdcfd 100644 --- a/README.md +++ b/README.md
@@ -57,6 +57,9 @@ - -u --url Gerrit server URL with the analytics plugins installed - -m --manifest Repo manifest XML path. Absolute path of the Repo manifest XML to import project from. Each project will be imported with the branch specified in the `revision` attribute. +- -n --manifest-branch (*optional*) Manifest branch. Manifest file git branch. +- -l --manifest-label (*optional*) Manifest label. A `manifest_label` is an aggregation of projects imported from the same manifest. +Add it to allow filtering by `manifest_label`. - -p --prefix (*optional*) Projects prefix. Limit the results to those projects that start with the specified prefix. - -e --elasticIndex Elastic Search index name. If not provided no ES export will be performed. _Note: ElastiSearch 6.x requires this index format `name/type`, while from ElasticSearch 7.x just `name`_
diff --git a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/engine/GerritAnalyticsTransformations.scala b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/engine/GerritAnalyticsTransformations.scala index c8cdaed..fab54a4 100644 --- a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/engine/GerritAnalyticsTransformations.scala +++ b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/engine/GerritAnalyticsTransformations.scala
@@ -189,6 +189,11 @@ .handleAliases(aliasesDFMaybe) .dropCommits } + + def addManifestInfo(manifestLabel: String, manifestBranch: Option[String])(implicit spark: SparkSession): DataFrame = { + val dfWithManifestInfo = df.withColumn("manifest_label", lit(manifestLabel)) + manifestBranch.fold(dfWithManifestInfo)(mb => dfWithManifestInfo.withColumn("manifest_branch", lit(mb))) + } } private def emailToDomain(email: String): String = email match {
diff --git a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala index 463dd28..b3a737e 100644 --- a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala +++ b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala
@@ -90,6 +90,14 @@ c.copy(manifest = Some(x)) } text "repo manifest XML with the list of projects to process" + opt[String]('n', "manifest-branch") optional () action { (input, c) => + c.copy(manifestBranch = Some(input)) + } text "manifest file git branch" + + opt[String]('l', "manifest-label") optional () action { (input, c) => + c.copy(manifestLabel = Some(input)) + } text "a 'manifest-label' is an aggregation of projects imported from the same manifest. Add to allow query by 'manifest-label'." + } cliOptionParser.parse(args, GerritEndpointConfig()) match { @@ -132,7 +140,10 @@ config.contributorsUrl, config.gerritApiConnection ) - contributorsStats.dashboardStats(aliasesDF) + val df = contributorsStats.dashboardStats(aliasesDF) + config.manifest.flatMap(_ => + config.manifestLabel.map(p => df.addManifestInfo(p, config.manifestBranch))) + .getOrElse(df) } def saveES(df: DataFrame)(implicit config: GerritEndpointConfig) {
diff --git a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/model/GerritEndpointConfig.scala b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/model/GerritEndpointConfig.scala index 6b0c678..53e26c7 100644 --- a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/model/GerritEndpointConfig.scala +++ b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/model/GerritEndpointConfig.scala
@@ -37,7 +37,9 @@ password: Option[String] = None, ignoreSSLCert: Option[Boolean] = None, extractBranches: Option[Boolean] = None, - manifest: Option[String] = None + manifest: Option[String] = None, + manifestBranch: Option[String] = None, + manifestLabel: Option[String] = None ) { lazy val projectsFromManifest: Option[Set[GerritProjectWithRef]] = manifest.map { mf =>
diff --git a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala index 28e4010..d49063b 100644 --- a/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala +++ b/gitcommits/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala
@@ -79,6 +79,20 @@ ) var manifest: String = null + @ArgOption( + name = "--manifest-branch", + aliases = Array("-n"), + usage = "manifest file git branch" + ) + var manifestBranch: String = null + + @ArgOption( + name = "--manifest-label", + aliases = Array("-l"), + usage = "A 'manifest_label' is an aggregation of projects imported from the same manifest. Add it to allow filtering by 'manifest_label'" + ) + var manifestLabel: String = null + override def run() { implicit val config = GerritEndpointConfig( gerritConfig.getListenUrl(),
diff --git a/gitcommits/src/test/scala/com/gerritforge/analytics/GerritAnalyticsTransformationsSpec.scala b/gitcommits/src/test/scala/com/gerritforge/analytics/GerritAnalyticsTransformationsSpec.scala index 94336c8..c8ff288 100644 --- a/gitcommits/src/test/scala/com/gerritforge/analytics/GerritAnalyticsTransformationsSpec.scala +++ b/gitcommits/src/test/scala/com/gerritforge/analytics/GerritAnalyticsTransformationsSpec.scala
@@ -316,6 +316,65 @@ df.collect should contain theSameElementsAs expectedDF.collect } + "addManifestInfo" should "enrich the data with manifest-label from the configuration" in { + import spark.implicits._ + + val manifestLabel = "testManifestLabel" + val inputDF = sc + .parallelize( + Seq( + ("input_value1"), + ("input_value2") + ) + ) + .toDF("input") + + val expectedDF = sc + .parallelize( + Seq( + ("input_value1", manifestLabel), + ("input_value2", manifestLabel) + ) + ) + .toDF("input", "manifest_label") + + val df = inputDF.addManifestInfo(manifestLabel, None) + + df.schema.fields.map(_.name) should contain allOf ("input", "manifest_label") + + df.collect should contain theSameElementsAs expectedDF.collect + } + + it should "enrich the data with manifest-label and manifest branch from the configuration" in { + import spark.implicits._ + + val manifestLabel = "testManifestLabel" + val manifestBranch = "manifestBranch" + val inputDF = sc + .parallelize( + Seq( + ("input_value1"), + ("input_value2") + ) + ) + .toDF("input") + + val expectedDF = sc + .parallelize( + Seq( + ("input_value1", manifestLabel, manifestBranch), + ("input_value2", manifestLabel, manifestBranch) + ) + ) + .toDF("input", "manifest_label", "manifest_branch") + + val df = inputDF.addManifestInfo(manifestLabel, Some(manifestBranch)) + + df.schema.fields.map(_.name) should contain allOf ("input", "manifest_label", "manifest_branch") + + df.collect should contain theSameElementsAs expectedDF.collect + } + it should "enrich the data with organization from the alias DF when available" in { import spark.implicits._