Add spark ETL container to wizard docker-compose
When setting up the docker-compose used to start up the wizard stack
(elasticsearch, kibabna, dashboard-importer), now the spark ETL job
is also included in the mix:
A `docker-compose up` will now also trigger the ETL job.
The ETL job waits for elasticsearch to be available before attempting
to import data to it.
In order for gerrit analytics endpoint to be available to the
dockerized ETL job, the host ip address and gerrit port need
to be configured in the docker compose:
- The host ip address is read using InetAddress object
- The gerrit schema and port are read from configuration
Feature: Issue 9867
Change-Id: I3419a635d8d7333e4d2ea078daec3e19287cbac0
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetup.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetup.scala
index 6e5da17..fc53340 100644
--- a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetup.scala
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetup.scala
@@ -13,6 +13,7 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.analytics.wizard
+import java.net.URL
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}
@@ -26,7 +27,7 @@
}
}
-case class AnalyticDashboardSetup(name: String, dockerComposeYamlPath: Path)(
+case class AnalyticDashboardSetup(name: String, dockerComposeYamlPath: Path, gerritLocalUrl: URL)(
implicit val writer: ConfigWriter) {
// Docker doesn't like container names with '/', hence the replace with '-'
@@ -40,6 +41,21 @@
|version: '3'
|services:
|
+ | spark-gerrit-analytics-etl:
+ | extra_hosts:
+ | - gerrit:${gerritLocalUrl.getHost}
+ | image: gerritforge/spark-gerrit-analytics-etl:latest
+ | environment:
+ | - ES_HOST=elasticsearch
+ | - GERRIT_URL=${gerritLocalUrl.getProtocol}://gerrit:${gerritLocalUrl.getPort}
+ | - ANALYTICS_ARGS=--since 2000-06-01 --aggregate email_hour --writeNotProcessedEventsTo file:///tmp/failed-events -e gerrit/analytics
+ | networks:
+ | - ek
+ | links:
+ | - elasticsearch
+ | depends_on:
+ | - elasticsearch
+ |
| dashboard-importer:
| image: gerritforge/analytics-dashboard-importer:latest
| networks:
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala
index 98bb03f..baf08fb 100644
--- a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala
@@ -19,41 +19,46 @@
import com.google.common.io.ByteStreams
import com.google.gerrit.extensions.annotations.PluginData
-import com.google.gerrit.extensions.restapi.{
- Response,
- RestApiException,
- RestModifyView,
- RestReadView
-}
+import com.google.gerrit.extensions.restapi.{Response, RestApiException, RestModifyView, RestReadView}
+import com.google.gerrit.server.config.GerritServerConfig
import com.google.gerrit.server.project.ProjectResource
import com.google.inject.{ImplementedBy, Inject}
import com.googlesource.gerrit.plugins.analytics.wizard.AnalyticDashboardSetup.writer
-import com.spotify.docker.client.{DefaultDockerClient, DockerClient}
+import com.googlesource.gerrit.plugins.analytics.wizard.utils._
import com.spotify.docker.client.messages.ContainerInfo
+import com.spotify.docker.client.{DefaultDockerClient, DockerClient}
+import org.eclipse.jgit.lib.Config
+
+import scala.util.{Failure, Success}
class Input(var dashboardName: String)
-class PutAnalyticsStack @Inject()(@PluginData val dataPath: Path)
+class PutAnalyticsStack @Inject()(@PluginData val dataPath: Path, @GerritServerConfig gerritConfig: Config)
extends RestModifyView[ProjectResource, Input] {
- override def apply(resource: ProjectResource,
- input: Input): Response[String] = {
+
+ override def apply(resource: ProjectResource, input: Input): Response[String] = {
val projectName = resource.getName
- val encodedName = AnalyticsWizardActions
- .encodedName(projectName)
+ val encodedName = AnalyticsWizardActions.encodedName(projectName)
- AnalyticDashboardSetup(
- projectName,
- dataPath.resolve(s"docker-compose.${encodedName}.yaml"))
- .createDashboardSetupFile()
- Response.created(s"Dashboard configuration created for $encodedName!")
+ val configHelper = new GerritConfigHelper(gerritConfig) with LocalAddressGetter
+ configHelper.getGerritLocalAddress match {
+ case Success(gerritLocalUrl) =>
+ AnalyticDashboardSetup(
+ projectName, dataPath.resolve(s"docker-compose.$encodedName.yaml"), gerritLocalUrl
+ ).createDashboardSetupFile()
+
+ Response.created(s"Dashboard configuration created for $encodedName!")
+ case Failure(exception) =>
+ Response.withStatusCode(500, s"Cannot create dashboard configuration - '${exception.getMessage}'")
+ }
}
}
class DockerComposeCommand(var action: String)
class PostAnalyticsStack @Inject()(@PluginData val dataPath: Path)
- extends RestModifyView[ProjectResource, DockerComposeCommand] {
+extends RestModifyView[ProjectResource, DockerComposeCommand] {
override def apply(resource: ProjectResource,
input: DockerComposeCommand): Response[String] = {
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/utils/GerritConfigHelper.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/utils/GerritConfigHelper.scala
new file mode 100644
index 0000000..cd7e946
--- /dev/null
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/utils/GerritConfigHelper.scala
@@ -0,0 +1,19 @@
+package com.googlesource.gerrit.plugins.analytics.wizard.utils
+
+import java.net.{InetAddress, URL}
+
+import org.eclipse.jgit.lib.Config
+
+import scala.util.Try
+
+class GerritConfigHelper(gerritConfig: Config) { self: LocalAddressGetter =>
+
+ def getGerritLocalAddress: Try[URL] = for {
+ listenUrl <- Try { new URL(gerritConfig.getString("httpd", null, "listenUrl")) }
+ localAddress <- Try { getLocalAddress }
+ } yield new URL(s"${listenUrl.getProtocol}://$localAddress:${listenUrl.getPort}")
+}
+
+trait LocalAddressGetter {
+ def getLocalAddress = InetAddress.getLocalHost.getHostAddress
+}
\ No newline at end of file
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetupSpec.scala b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetupSpec.scala
index c7e0519..d539469 100644
--- a/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetupSpec.scala
+++ b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticDashboardSetupSpec.scala
@@ -1,6 +1,7 @@
package com.googlesource.gerrit.plugins.analytics.wizard
import java.io.File
+import java.net.URL
import java.nio.file.Path
import org.scalatest.{FlatSpec, Matchers}
@@ -18,7 +19,7 @@
implicit val writer = new MockWriter()
val composeYamlFile = File.createTempFile(getClass.getName, ".yaml").toPath
- val ads = AnalyticDashboardSetup("aProject", composeYamlFile)
+ val ads = AnalyticDashboardSetup("aProject", composeYamlFile, new URL("http://gerrit_local_ip_address:8080"))
ads.createDashboardSetupFile()
gotFilename shouldBe Some(composeYamlFile)
}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/fixtures/TestFixtures.scala b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/fixtures/TestFixtures.scala
new file mode 100644
index 0000000..9ccfcdd
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/fixtures/TestFixtures.scala
@@ -0,0 +1,16 @@
+package com.googlesource.gerrit.plugins.analytics.wizard.fixtures
+
+import org.eclipse.jgit.lib.Config
+
+trait TestFixtures {
+
+ def gerritConfig(httpProtocol: String, httpPort: Int): Config = {
+ val gerritConfig = new Config()
+ gerritConfig.fromText(
+ s"""
+ |[httpd]
+ | listenUrl = $httpProtocol://*:$httpPort/
+ """.stripMargin)
+ gerritConfig
+ }
+}
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/utils/GerritConfigHelperSpec.scala b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/utils/GerritConfigHelperSpec.scala
new file mode 100644
index 0000000..264400e
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/utils/GerritConfigHelperSpec.scala
@@ -0,0 +1,41 @@
+package com.googlesource.gerrit.plugins.analytics.wizard.utils
+
+import java.net.{MalformedURLException, URL}
+
+import com.googlesource.gerrit.plugins.analytics.wizard.fixtures.TestFixtures
+import org.eclipse.jgit.lib.Config
+import org.scalatest.TryValues._
+import org.scalatest.{FlatSpec, Matchers}
+
+class GerritConfigHelperSpec extends FlatSpec with Matchers with TestFixtures {
+ behavior of "getGerritLocalAddress"
+
+ it should "retrieve a URL successfully when the right configuration is set" in {
+ val httpProtocol = "http"
+ val httpPort = 8080
+ val helper = new GerritConfigHelper(gerritConfig(httpProtocol, httpPort)) with TestLocalAddressGetter
+
+ helper.getGerritLocalAddress.success.value shouldBe new URL(s"$httpProtocol://${helper.getLocalAddress}:$httpPort")
+
+ }
+
+ it should "fail in retrieving a URL when configuration is missing" in {
+ val helper = new GerritConfigHelper(new Config()) with TestLocalAddressGetter
+
+ helper.getGerritLocalAddress.failure.exception shouldBe a[MalformedURLException]
+ }
+
+ it should "fail in retrieving a URL when local address cannot be retrieved" in {
+ val helper = new GerritConfigHelper(new Config()) with FailingLocalAddressGetter
+
+ helper.getGerritLocalAddress.failure.exception shouldBe a[MalformedURLException]
+ }
+}
+
+trait TestLocalAddressGetter extends LocalAddressGetter {
+ override def getLocalAddress: String = "127.0.0.1"
+}
+
+trait FailingLocalAddressGetter extends LocalAddressGetter {
+ override def getLocalAddress: String = throw new Exception("Cannot get local address")
+}
\ No newline at end of file
