Only pass index name to ETL
Index type will be inferred by the analytics imported by the job.
At the moment we just hardcoded 'gitcommits', but in future we will have more (i.e.: auditlogs)
Feature: Issue 9984
Change-Id: I15d498ad3519971efc362a4020521dc6ce020b45
diff --git a/README.md b/README.md
index 44c9229..2d9c25e 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,8 @@
--url http://gerrit.mycompany.com \
--events file:///tmp/gerrit-events-export.json \
--writeNotProcessedEventsTo file:///tmp/failed-events \
- -e gerrit/analytics \
+ -e gerrit
+ \
--username gerrit-api-username \
--password gerrit-api-password
```
@@ -28,7 +29,7 @@
docker run -ti --rm \
-e ES_HOST="es.mycompany.com" \
-e GERRIT_URL="http://gerrit.mycompany.com" \
- -e ANALYTICS_ARGS="--since 2000-06-01 --aggregate email_hour --writeNotProcessedEventsTo file:///tmp/failed-events -e gerrit/analytics" \
+ -e ANALYTICS_ARGS="--since 2000-06-01 --aggregate email_hour --writeNotProcessedEventsTo file:///tmp/failed-events -e gerrit" \
gerritforge/spark-gerrit-analytics-etl:latest
```
@@ -39,8 +40,7 @@
see: https://gerrit.googlesource.com/plugins/analytics/+/master/README.md
- -u --url Gerrit server URL with the analytics plugins installed
- -p --prefix (*optional*) Projects prefix. Limit the results to those projects that start with the specified prefix.
-- -e --elasticIndex specify as <index>/<type> to be loaded in Elastic Search
- if not provided no ES export will be performed
+- -e --elasticIndex Elastic Search index name. If not provided no ES export will be performed
- -o --out folder location for storing the output as JSON files
if not provided data is saved to </tmp>/analytics-<NNNN> where </tmp> is
the system temporary directory
@@ -97,7 +97,7 @@
--network analytics-etl_ek \
-e ES_HOST="elasticsearch" \
-e GERRIT_URL="http://$HOST_IP:8080" \
- -e ANALYTICS_ARGS="--since 2000-06-01 --aggregate email_hour --writeNotProcessedEventsTo file:///tmp/failed-events -e gerrit/analytics" \
+ -e ANALYTICS_ARGS="--since 2000-06-01 --aggregate email_hour --writeNotProcessedEventsTo file:///tmp/failed-events -e gerrit" \
gerritforge/spark-gerrit-analytics-etl:latest
```
diff --git a/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala b/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala
index 16a376a..f99a4d1 100644
--- a/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala
+++ b/src/main/scala/com/gerritforge/analytics/gitcommits/job/Main.scala
@@ -137,6 +137,8 @@
self: LazyLogging with FetchProjects =>
implicit val codec = Codec.ISO8859
+ val indexType = "gitCommits"
+
def buildProjectStats()(implicit config: GerritEndpointConfig, spark: SparkSession): DataFrame = {
import com.gerritforge.analytics.gitcommits.engine.GerritAnalyticsTransformations._
import com.gerritforge.analytics.gitcommits.engine.events.GerritEventsTransformations._
@@ -241,9 +243,9 @@
import org.elasticsearch.spark.sql._
config.elasticIndex.foreach { esIndex =>
logger.info(
- s"ES content created, saving it to elastic search instance at '${config.elasticIndex}'")
+ s"ES content created, saving it to elastic search instance at '${config.elasticIndex}/$indexType'")
- df.saveToEs(esIndex)
+ df.saveToEs(s"$esIndex/$indexType")
}
}
diff --git a/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala b/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala
index 883b1f8..41cce5e 100644
--- a/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala
+++ b/src/main/scala/com/gerritforge/analytics/gitcommits/plugin/ProcessGitCommitsCommand.scala
@@ -5,11 +5,7 @@
import com.gerritforge.analytics.gitcommits.engine.events.AggregationStrategy
import com.gerritforge.analytics.gitcommits.job.{FetchProjects, Job}
-import com.gerritforge.analytics.gitcommits.model.{
- GerritEndpointConfig,
- GerritProject,
- GerritProjectsSupport
-}
+import com.gerritforge.analytics.gitcommits.model.{GerritEndpointConfig, GerritProject, GerritProjectsSupport}
import com.google.gerrit.server.project.ProjectControl
import com.google.gerrit.sshd.{CommandMetaData, SshCommand}
import com.google.inject.Inject
@@ -35,7 +31,7 @@
var projectControl: ProjectControl = null
@ArgOption(name = "--elasticIndex", aliases = Array("-e"), usage = "index name")
- var elasticIndex: String = "gerrit/analytics"
+ var elasticIndex: String = "gerrit"
@ArgOption(name = "--since", aliases = Array("-s"), usage = "begin date")
var beginDate: Timestamp = NO_TIMESTAMP
@@ -91,9 +87,9 @@
import org.elasticsearch.spark.sql._
config.elasticIndex.foreach { esIndex =>
stdout.println(
- s"$numRows rows extracted. Posting Elasticsearch at '${config.elasticIndex}'")
+ s"$numRows rows extracted. Posting Elasticsearch at '${config.elasticIndex}/$indexType'")
stdout.flush()
- projectStats.saveToEs(esIndex)
+ projectStats.saveToEs(s"$esIndex/$indexType")
}
val elaspsedTs = (System.currentTimeMillis - startTs) / 1000L
