Receive ETL configuration parameter
Modify 'PUT analytics-wizard~stack' to receive the ETL configuration
paramenters.
Feature: Issue 9868
Change-Id: I57790c40937de8d2f485e887c18185f9873c3634
diff --git a/build.sbt b/build.sbt
index f382090..50f2883 100644
--- a/build.sbt
+++ b/build.sbt
@@ -3,6 +3,8 @@
val gerritApiVersion = "2.16-rc0"
val pluginName = "analytics-wizard"
+scalaVersion := "2.11.12"
+
git.useGitDescribe := true
scalafmtOnCompile in ThisBuild := true
@@ -16,6 +18,7 @@
"com.google.inject" % "guice" % "3.0" % Provided,
"com.google.gerrit" % "gerrit-plugin-api" % gerritApiVersion % Provided withSources (),
"com.spotify" % "docker-client" % "8.14.1",
+ "com.beachape" %% "enumeratum" % "1.5.13",
"org.scalatest" %% "scalatest" % "3.0.4" % Test,
"net.codingwell" %% "scala-guice" % "4.1.0" % Test
),
diff --git a/src/main/resources/static/js/analyticswizard.js b/src/main/resources/static/js/analyticswizard.js
index 7bbb2cf..565a7d7 100644
--- a/src/main/resources/static/js/analyticswizard.js
+++ b/src/main/resources/static/js/analyticswizard.js
@@ -49,12 +49,18 @@
function submitDetailsForm() {
var projectName = encodeURIComponent($("#input-project-name").val());
+ var requestBody = {
+ dashboard_name: projectName,
+ etl_config: {
+ aggregate: "email"
+ }
+ }
$.ajax({
type : "PUT",
url : `/a/projects/${projectName}/analytics-wizard~stack`,
dataType: 'application/json',
// Initially project-dashboard is a 1 to 1 relationship
- data: "{'dashboard_name': '" + projectName + "'}",
+ data: JSON.stringify(requestBody),
contentType:"application/json; charset=utf-8",
// Need to catch the status code since Gerrit doesn't return
// a well formed JSON, hence Ajax treats it as an error
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala
index e424681..3eba31c 100644
--- a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/AnalyticsWizardActions.scala
@@ -12,11 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.googlesource.gerrit.plugins.analytics.wizard
-
import java.net.URLEncoder
import java.nio.charset.StandardCharsets.UTF_8
import java.nio.file.Path
-
import com.google.common.io.ByteStreams
import com.google.gerrit.extensions.annotations.PluginData
import com.google.gerrit.extensions.restapi.{
@@ -29,52 +27,54 @@
import com.google.gerrit.server.project.ProjectResource
import com.google.inject.{ImplementedBy, Inject}
import com.googlesource.gerrit.plugins.analytics.wizard.AnalyticDashboardSetup.writer
+import com.googlesource.gerrit.plugins.analytics.wizard.model.{
+ ETLConfig,
+ ETLConfigRaw,
+ ETLConfigValidationError
+}
import com.googlesource.gerrit.plugins.analytics.wizard.utils._
import com.spotify.docker.client.messages.ContainerInfo
import com.spotify.docker.client.{DefaultDockerClient, DockerClient}
import org.eclipse.jgit.lib.Config
-
import scala.util.{Failure, Success}
-
-class Input(var dashboardName: String)
-
+case class Input(dashboardName: String, etlConfig: ETLConfigRaw)
class PutAnalyticsStack @Inject()(@PluginData val dataPath: Path,
@GerritServerConfig gerritConfig: Config)
extends RestModifyView[ProjectResource, Input] {
-
override def apply(resource: ProjectResource, input: Input): Response[String] = {
-
- val projectName = resource.getName
- val encodedName = AnalyticsWizardActions.encodedName(projectName)
-
- val configHelper = new GerritConfigHelper(gerritConfig) with LocalAddressGetter
-
- configHelper.getGerritLocalAddress match {
- case Success(gerritLocalUrl) =>
- AnalyticDashboardSetup(
- projectName,
- dataPath.resolve(s"docker-compose.$encodedName.yaml"),
- gerritLocalUrl
- ).createDashboardSetupFile()
-
- Response.created(s"Dashboard configuration created for $encodedName!")
- case Failure(exception) =>
- Response.withStatusCode(
- 500,
- s"Cannot create dashboard configuration - '${exception.getMessage}'")
- }
+ val projectName = resource.getName
+ val encodedName = AnalyticsWizardActions.encodedName(projectName)
+ val etlConfigE: Either[ETLConfigValidationError, ETLConfig] = ETLConfig.fromRaw(input.etlConfig)
+ etlConfigE.fold(
+ configError =>
+ Response.withStatusCode(400,
+ s"Cannot create dashboard configuration: ${configError.message}"),
+ etlConfig => {
+ val configHelper = new GerritConfigHelper(gerritConfig) with LocalAddressGetter
+ configHelper.getGerritLocalAddress match {
+ case Success(gerritLocalUrl) =>
+ AnalyticDashboardSetup(
+ projectName,
+ dataPath.resolve(s"docker-compose.$encodedName.yaml"),
+ gerritLocalUrl
+ ).createDashboardSetupFile()
+ Response.created(s"Dashboard configuration created for $encodedName!")
+ case Failure(exception) =>
+ Response.withStatusCode(
+ 500,
+ s"Cannot create dashboard configuration - '${exception.getMessage}'")
+ }
+ }
+ )
}
}
-
class DockerComposeCommand(var action: String)
class PostAnalyticsStack @Inject()(@PluginData val dataPath: Path)
extends RestModifyView[ProjectResource, DockerComposeCommand] {
override def apply(resource: ProjectResource, input: DockerComposeCommand): Response[String] = {
-
val projectName = resource.getName
val encodedName = AnalyticsWizardActions
.encodedName(projectName)
-
val pb = new ProcessBuilder(
"docker-compose",
"-f",
@@ -83,13 +83,11 @@
"--detach"
)
pb.redirectErrorStream(true)
-
val ps: Process = pb.start
ps.getOutputStream.close
val output =
new String(ByteStreams.toByteArray(ps.getInputStream), UTF_8)
ps.waitFor
-
ps.exitValue match {
case 0 => Response.created(output)
case _ =>
@@ -97,7 +95,6 @@
}
}
}
-
class GetAnalyticsStackStatus @Inject()(@PluginData val dataPath: Path,
val dockerClientProvider: DockerClientProvider)
extends RestReadView[ProjectResource] {
@@ -105,7 +102,6 @@
val containerName = "analytics-wizard_spark-gerrit-analytics-etl_1"
responseFromContainerInfo(dockerClientProvider.client.inspectContainer(containerName))
}
-
private def responseFromContainerInfo(containerInfo: ContainerInfo) = {
containerInfo.state match {
case s if s.exitCode != 0 =>
@@ -119,7 +115,6 @@
}
}
}
-
object AnalyticsWizardActions {
// URLEncoder could potentially throw UnsupportedEncodingException,
// but UTF-8 will *always* be resolved, otherwise, Gerrit wouldn't work at all
@@ -130,12 +125,10 @@
case e: Throwable => throw new RuntimeException(e)
}
}
-
@ImplementedBy(classOf[DockerClientProviderImpl])
trait DockerClientProvider {
def client: DockerClient
}
-
class DockerClientProviderImpl extends DockerClientProvider {
def client: DockerClient = DefaultDockerClient.fromEnv.build
}
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfig.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfig.scala
new file mode 100644
index 0000000..11a1ab2
--- /dev/null
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfig.scala
@@ -0,0 +1,114 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.googlesource.gerrit.plugins.analytics.wizard.model
+
+import java.net.URL
+import java.time.LocalDate
+import java.time.format.DateTimeFormatter
+
+import com.googlesource.gerrit.plugins.analytics.wizard.model.AggregationType.Email
+import enumeratum._
+
+import scala.util.{Failure, Success, Try}
+
+case class ETLConfig(aggregate: AggregationType,
+ projectPrefix: Option[String],
+ since: Option[LocalDate],
+ until: Option[LocalDate],
+ eventsUrl: Option[URL],
+ writeNotProcessedEventsTo: Option[URL],
+ emailAliasesPath: Option[String],
+ username: Option[String],
+ password: Option[String])
+
+sealed trait AggregationType extends EnumEntry
+object AggregationType extends Enum[AggregationType] {
+ val values = findValues
+
+ case object Email extends AggregationType
+ case object EmailHour extends AggregationType
+ case object EmailDay extends AggregationType
+ case object EmailMonth extends AggregationType
+ case object EmailYear extends AggregationType
+}
+
+object ETLConfig {
+ def fromRaw(raw: ETLConfigRaw): Either[ETLConfigValidationError, ETLConfig] = {
+ for {
+ s <- validateLocalDate("since", raw.since).right
+ u <- validateLocalDate("until", raw.until).right
+ w <- validateUrl("writeNotProcessedEventsTo", raw.writeNotProcessedEventsTo).right
+ eu <- validateUrl("eventsUrl", raw.eventsUrl).right
+ a <- validateAggregate(raw.aggregate).right
+ } yield
+ ETLConfig(
+ aggregate = a,
+ projectPrefix = Option(raw.projectPrefix),
+ since = s,
+ until = u,
+ eventsUrl = eu,
+ writeNotProcessedEventsTo = w,
+ emailAliasesPath = Option(raw.emailAliasesPath),
+ username = Option(raw.username),
+ password = Option(raw.password)
+ )
+ }
+
+ private def validateLocalDate(
+ parameter: String,
+ value: String): Either[LocalDateValidationError, Option[LocalDate]] = {
+ val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd")
+
+ Option(value)
+ .map { maybeDate =>
+ Try(LocalDate.parse(maybeDate, formatter)) match {
+ case Success(date) => Right(Some(date))
+ case Failure(exception) => Left(LocalDateValidationError(parameter, maybeDate, exception))
+ }
+ }
+ .getOrElse(Right(None))
+ }
+ private def validateUrl(parameter: String,
+ value: String): Either[UrlValidationError, Option[URL]] = {
+ Option(value)
+ .map { u =>
+ Try(new URL(u)) match {
+ case Success(url) => Right(Some(url))
+ case Failure(exception) => Left(UrlValidationError(parameter, u, exception))
+ }
+ }
+ .getOrElse(Right(None))
+ }
+ private def validateAggregate(
+ value: String): Either[AggregateValidationError, AggregationType] = {
+ val maybeAggregate =
+ AggregationType.withNameInsensitiveOption(Option(value).getOrElse("email").replace("_", ""))
+ Either.cond(maybeAggregate.isDefined, maybeAggregate.get, AggregateValidationError(value))
+ }
+}
+
+sealed trait ETLConfigValidationError {
+ def message: String = s"Error validating '$parameter' parameter: $value. Exception: $cause"
+ def value: String
+ def parameter: String
+ def cause: Throwable
+}
+case class LocalDateValidationError(parameter: String, value: String, cause: Throwable)
+ extends ETLConfigValidationError
+case class UrlValidationError(parameter: String, value: String, cause: Throwable)
+ extends ETLConfigValidationError
+case class AggregateValidationError(value: String) extends ETLConfigValidationError {
+ val parameter = "aggregate"
+ val cause: Throwable = new Throwable(s"Value $value is not a valid aggregation type")
+}
diff --git a/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfigRaw.scala b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfigRaw.scala
new file mode 100644
index 0000000..1a16df9
--- /dev/null
+++ b/src/main/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfigRaw.scala
@@ -0,0 +1,24 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.googlesource.gerrit.plugins.analytics.wizard.model
+
+case class ETLConfigRaw(aggregate: String,
+ projectPrefix: String,
+ since: String,
+ until: String,
+ eventsUrl: String,
+ emailAliasesPath: String,
+ writeNotProcessedEventsTo: String,
+ username: String,
+ password: String)
diff --git a/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfigSpec.scala b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfigSpec.scala
new file mode 100644
index 0000000..7f9f0e5
--- /dev/null
+++ b/src/test/scala/com/googlesource/gerrit/plugins/analytics/wizard/model/ETLConfigSpec.scala
@@ -0,0 +1,150 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.googlesource.gerrit.plugins.analytics.wizard.model
+
+import java.net.{MalformedURLException, URL}
+import java.time.LocalDate
+import java.time.format.DateTimeParseException
+
+import org.scalatest.EitherValues._
+import org.scalatest.{FlatSpec, Matchers}
+
+class ETLConfigSpec extends FlatSpec with Matchers {
+
+ behavior of "ETLConfig.fromRaw"
+
+ val etlConfigRaw = ETLConfigRaw(
+ aggregate = "email",
+ projectPrefix = "prefix",
+ since = "2018-10-10",
+ until = "2018-10-13",
+ eventsUrl = "file:///tmp/gerrit-events-export.json",
+ emailAliasesPath = "/tmp/path",
+ writeNotProcessedEventsTo = "file://tmp/myfile.json",
+ username = "dss",
+ password = "dsd"
+ )
+
+ it should "pass validation with correct parameters" in {
+
+ ETLConfig.fromRaw(etlConfigRaw).right.value shouldBe ETLConfig(
+ aggregate = AggregationType.Email,
+ projectPrefix = Some(etlConfigRaw.projectPrefix),
+ since = Some(LocalDate.of(2018, 10, 10)),
+ until = Some(LocalDate.of(2018, 10, 13)),
+ eventsUrl = Some(new URL(etlConfigRaw.eventsUrl)),
+ writeNotProcessedEventsTo = Some(new URL(etlConfigRaw.writeNotProcessedEventsTo)),
+ emailAliasesPath = Some(etlConfigRaw.emailAliasesPath),
+ username = Some(etlConfigRaw.username),
+ password = Some(etlConfigRaw.password)
+ )
+ }
+
+ it should "defaults non mandatory parameters" in {
+ ETLConfig
+ .fromRaw(
+ etlConfigRaw.copy(projectPrefix = null,
+ since = null,
+ until = null,
+ eventsUrl = null,
+ writeNotProcessedEventsTo = null,
+ emailAliasesPath = null,
+ username = null,
+ password = null))
+ .right
+ .value shouldBe ETLConfig(
+ aggregate = AggregationType.Email,
+ projectPrefix = None,
+ since = None,
+ until = None,
+ eventsUrl = None,
+ writeNotProcessedEventsTo = None,
+ emailAliasesPath = None,
+ username = None,
+ password = None
+ )
+ }
+
+ it should "handle `null` mandatory parameters" in {
+ ETLConfig
+ .fromRaw(
+ etlConfigRaw.copy(
+ aggregate = null,
+ projectPrefix = null,
+ since = null,
+ until = null,
+ eventsUrl = null,
+ writeNotProcessedEventsTo = null,
+ emailAliasesPath = null,
+ username = null,
+ password = null
+ ))
+ .right
+ .value shouldBe ETLConfig(
+ aggregate = AggregationType.Email,
+ projectPrefix = None,
+ since = None,
+ until = None,
+ eventsUrl = None,
+ writeNotProcessedEventsTo = None,
+ emailAliasesPath = None,
+ username = None,
+ password = None
+ )
+ }
+
+ it should "fail validation for invalid 'since' parameter" in {
+ val invalidDate = "randomString"
+ val error = ETLConfig
+ .fromRaw(etlConfigRaw.copy(since = invalidDate))
+ .left
+ .value
+ error.value shouldBe invalidDate
+ error.parameter shouldBe "since"
+ error.cause shouldBe a[DateTimeParseException]
+ }
+
+ it should "fail validation for invalid 'until' parameter" in {
+ val invalidDate = "randomString"
+ val error = ETLConfig
+ .fromRaw(etlConfigRaw.copy(until = invalidDate))
+ .left
+ .value
+ error.value shouldBe invalidDate
+ error.parameter shouldBe "until"
+ error.cause shouldBe a[DateTimeParseException]
+ }
+
+ it should "fail validation for invalid 'writeNotProcessedEventsTo' parameter" in {
+ val invalidUrl = "randomString"
+ val error = ETLConfig
+ .fromRaw(etlConfigRaw.copy(writeNotProcessedEventsTo = invalidUrl))
+ .left
+ .value
+ error.value shouldBe invalidUrl
+ error.parameter shouldBe "writeNotProcessedEventsTo"
+ error.cause shouldBe a[MalformedURLException]
+ }
+
+ it should "fail validation for invalid 'eventsPath' parameter" in {
+ val invalidUrl = "not|good.txt"
+ val error = ETLConfig
+ .fromRaw(etlConfigRaw.copy(eventsUrl = invalidUrl))
+ .left
+ .value
+ error.value shouldBe invalidUrl
+ error.parameter shouldBe "eventsUrl"
+ error.cause shouldBe a[MalformedURLException]
+ }
+}
