Transform http and ssh logs into audit logs
Tranformation of SSH and HTTP logs into auditlogs will permit
to analyse audit log data even before the plugin was enabled.
Feature: Issue 10307
Change-Id: I404b17b4e18d6f0aa648bd13a471e1bf8847c04f
diff --git a/BUILD b/BUILD
index 2dec1e5..db99a0a 100644
--- a/BUILD
+++ b/BUILD
@@ -13,6 +13,7 @@
"Gerrit-PluginName: audit-sl4j",
"Gerrit-ReloadMode: reload",
"Gerrit-Module: com.googlesource.gerrit.plugins.auditsl4j.Module",
+ "Gerrit-SshModule: com.googlesource.gerrit.plugins.auditsl4j.SshModule",
"Implementation-Title: Gerrit Audit provider for SLF4J",
"Implementation-URL: https://gerrit.googlesource.com/plugins/audit-sl4j/",
],
diff --git a/README.md b/README.md
index d4c4c19..7145605 100644
--- a/README.md
+++ b/README.md
@@ -103,3 +103,11 @@
}
}
```
+
+## Import old logs
+
+The makes available an ssh command to transform `http_logs` and `ssh_logs` into Audit Logs:
+
+```bash
+ssh -p 29418 admin@localhost audit-sl4j transform --from 2019-01-23 --until 2019-01-24
+```
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java
index 614b862..2507acc 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditFormatRenderer.java
@@ -15,12 +15,13 @@
package com.googlesource.gerrit.plugins.auditsl4j;
import com.google.gerrit.server.AuditEvent;
-import com.google.gerrit.server.audit.SshAuditEvent;
import java.util.Optional;
public interface AuditFormatRenderer {
String render(AuditEvent auditEvent);
+ String render(AuditEvent auditEvent, TransformableAuditLogType type);
+
Optional<String> headers();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java
index 85c4be5..794e820 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRecord.java
@@ -27,4 +27,11 @@
this.type = eventClass.substring(eventClass.lastIndexOf('.') + 1);
this.event = event;
}
+
+ public AuditRecord(AuditEvent event, TransformableAuditLogType type) {
+ super();
+
+ this.type = type.name();
+ this.event = event;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java
index 0dbbf74..4c162be 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToCsv.java
@@ -89,6 +89,11 @@
}
@Override
+ public String render(AuditEvent auditEvent, TransformableAuditLogType type) {
+ return render(auditEvent);
+ }
+
+ @Override
public Optional<String> headers() {
return Optional.of(
"EventId | EventTS | SessionId | User | Protocol data | Action | Parameters | Result | StartTS | Elapsed");
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java
index b0c8b47..5a21054 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditRendererToJson.java
@@ -17,9 +17,9 @@
import com.google.common.collect.ListMultimap;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.server.AccessPath;
+import com.google.gerrit.server.AuditEvent;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.OutputFormat;
-import com.google.gerrit.server.AuditEvent;
import com.google.gson.ExclusionStrategy;
import com.google.gson.FieldAttributes;
import com.google.gson.Gson;
@@ -74,6 +74,11 @@
}
@Override
+ public String render(AuditEvent auditEvent, TransformableAuditLogType type) {
+ return gson.toJson(new AuditRecord(auditEvent, type));
+ }
+
+ @Override
public Optional<String> headers() {
return Optional.empty();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditUser.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditUser.java
new file mode 100644
index 0000000..2e6dd8a
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditUser.java
@@ -0,0 +1,42 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.auditsl4j;
+
+import com.google.gerrit.server.CurrentUser;
+import com.google.gerrit.server.account.GroupMembership;
+import java.util.Optional;
+
+public class AuditUser extends CurrentUser {
+ String username;
+
+ @Override
+ public GroupMembership getEffectiveGroups() {
+ return null;
+ }
+
+ @Override
+ public Object getCacheKey() {
+ return null;
+ }
+
+ @Override
+ public Optional<String> getUserName() {
+ return Optional.of(username);
+ }
+
+ public void setUserName(String username) {
+ this.username = username;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java
index 1da9b2a..32866a0 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriter.java
@@ -14,7 +14,6 @@
package com.googlesource.gerrit.plugins.auditsl4j;
-import com.google.gerrit.server.audit.HttpAuditEvent;
import com.google.inject.ImplementedBy;
@ImplementedBy(AuditWriterToLogger.class)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java
index 98f64e2..51d4629 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToLogger.java
@@ -14,7 +14,6 @@
package com.googlesource.gerrit.plugins.auditsl4j;
-import com.google.gerrit.server.audit.SshAuditEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java
index 6aab77b..bb24056 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/AuditWriterToStringList.java
@@ -14,7 +14,6 @@
package com.googlesource.gerrit.plugins.auditsl4j;
-import com.google.gerrit.server.audit.HttpAuditEvent;
import com.google.inject.Singleton;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java
index 9d14f18..e09035e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/LoggerAudit.java
@@ -27,7 +27,7 @@
public static final String AUDIT_LOGGER_NAME = LoggerAudit.class.getName();
@Inject
- LoggerAudit(AuditWriter auditWriter, AuditFormatRenderer auditRenderer) {
+ public LoggerAudit(AuditWriter auditWriter, AuditFormatRenderer auditRenderer) {
this.auditWriter = auditWriter;
this.auditRenderer = auditRenderer;
@@ -36,7 +36,15 @@
@Override
public void onAuditableAction(AuditEvent auditEvent) {
- String auditString = auditRenderer.render(auditEvent);
+ String auditString = getAuditString(auditEvent);
auditWriter.write(auditString);
}
+
+ public String getAuditString(AuditEvent auditEvent) {
+ return auditRenderer.render(auditEvent);
+ }
+
+ public String getAuditString(AuditEvent auditEvent, TransformableAuditLogType type) {
+ return auditRenderer.render(auditEvent, type);
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/SshModule.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/SshModule.java
new file mode 100644
index 0000000..125b2fc
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/SshModule.java
@@ -0,0 +1,24 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.auditsl4j;
+
+import com.google.gerrit.sshd.PluginCommandModule;
+
+public class SshModule extends PluginCommandModule {
+ @Override
+ protected void configureCommands() {
+ command(TransformLogsCommand.class);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformLogsCommand.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformLogsCommand.java
new file mode 100644
index 0000000..6c5b463
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformLogsCommand.java
@@ -0,0 +1,149 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.auditsl4j;
+
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.sshd.CommandMetaData;
+import com.google.gerrit.sshd.SshCommand;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.auditsl4j.logsource.HTTPLog;
+import com.googlesource.gerrit.plugins.auditsl4j.logsource.SSHLog;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.zip.GZIPInputStream;
+import org.kohsuke.args4j.Option;
+
+@CommandMetaData(name = "transform", description = "Transform ssh and http logs into audit logs")
+public class TransformLogsCommand extends SshCommand {
+
+ private LoggerAudit loggerAudit;
+ private final SitePaths sitePaths;
+
+ @Inject
+ public TransformLogsCommand(SitePaths sitePaths, LoggerAudit loggerAudit) {
+ this.sitePaths = sitePaths;
+ this.loggerAudit = loggerAudit;
+ }
+
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+
+ @Option(name = "--from", usage = "transform logs from <YYYY-MM-DD>")
+ private String from;
+
+ @Option(name = "--until", usage = "transform logs until <YYYY-MM-DD>")
+ private String until;
+
+ @Override
+ public void run() {
+
+ Date dateFrom;
+ try {
+ dateFrom = format.parse(from);
+ } catch (Exception e) {
+ stderr.print("Invalid 'from' format: " + from + ", expected format <YYYY-MM-DD>");
+ return;
+ }
+ Date dateUntil;
+ try {
+ dateUntil = format.parse(until);
+ } catch (Exception e) {
+ stderr.print("Invalid 'until' format: " + until + ", expected format <YYYY-MM-DD>");
+ return;
+ }
+
+ if (dateFrom.after(dateUntil)) {
+ stderr.print("'from' cannot be after 'until'");
+ return;
+ }
+
+ Date currentDate = dateFrom;
+ while (currentDate.compareTo(dateUntil) <= 0) {
+ transformHttpdLogs(format.format(currentDate));
+ transformSshdLogs(format.format(currentDate));
+
+ currentDate = getTomorrowDate(currentDate);
+ }
+
+ stdout.print("Transformed HTTP and SSH logs from " + from + " until " + until + "!\n");
+ }
+
+ private Date getTomorrowDate(Date currentDate) {
+ Calendar c = Calendar.getInstance();
+ c.setTime(currentDate);
+ c.add(Calendar.DAY_OF_MONTH, 1);
+ return c.getTime();
+ }
+
+ private void transformSshdLogs(String currentDateString) {
+ transformLogs(currentDateString, SSHLog.logFilenameBase(), SSHLog::createFromLog);
+ }
+
+ private void transformHttpdLogs(String currentDateString) {
+ transformLogs(currentDateString, HTTPLog.logFilenameBase(), HTTPLog::createFromLog);
+ }
+
+ private void transformLogs(
+ String currentDateString,
+ String fileType,
+ Function<String, Optional<? extends TransformableLog>> createTransformable) {
+ // Log format example: httpd_log.2019-01-19.gz
+ String logFileName = sitePaths.logs_dir + "/" + fileType + "." + currentDateString + ".gz";
+ String auditLogFileName = sitePaths.logs_dir + "/audit_log." + currentDateString + ".log";
+
+ stdout.print("Transforming: " + logFileName + " => " + auditLogFileName + " ...\n");
+ stdout.flush();
+
+ try {
+ GZIPInputStream gzis = new GZIPInputStream(new FileInputStream(logFileName));
+ BufferedReader input = new BufferedReader(new InputStreamReader(gzis));
+
+ PrintWriter pw =
+ new PrintWriter(
+ Files.newBufferedWriter(
+ Paths.get(auditLogFileName),
+ StandardOpenOption.CREATE,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.APPEND));
+
+ input
+ .lines()
+ .map(createTransformable)
+ .map(
+ maybeTransformableLog ->
+ maybeTransformableLog.flatMap(
+ transformableLog -> transformableLog.toAuditLog(loggerAudit)))
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .forEach(pw::println);
+ pw.flush();
+ } catch (FileNotFoundException fnfe) {
+ stderr.print("Cannot find '" + logFileName + "'. Skipping!\n");
+ } catch (IOException e) {
+ stderr.print("Error: " + e.getMessage() + "!\n");
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableAuditLogType.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableAuditLogType.java
new file mode 100644
index 0000000..8a2b589
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableAuditLogType.java
@@ -0,0 +1,21 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.auditsl4j;
+
+public enum TransformableAuditLogType {
+ HttpAuditEvent,
+ ExtendedHttpAuditEvent,
+ SshAuditEvent
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableLog.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableLog.java
new file mode 100644
index 0000000..8c07866
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/TransformableLog.java
@@ -0,0 +1,21 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.auditsl4j;
+
+import java.util.Optional;
+
+public interface TransformableLog {
+ Optional<String> toAuditLog(LoggerAudit loggerAudit);
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HTTPLog.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HTTPLog.java
new file mode 100644
index 0000000..781e8bc
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HTTPLog.java
@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.googlesource.gerrit.plugins.auditsl4j.logsource;
+
+import static com.google.gerrit.server.AuditEvent.UNKNOWN_SESSION_ID;
+
+import com.google.gerrit.server.AccessPath;
+import com.google.gerrit.server.audit.HttpAuditEvent;
+import com.googlesource.gerrit.plugins.auditsl4j.AuditUser;
+import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit;
+import com.googlesource.gerrit.plugins.auditsl4j.TransformableAuditLogType;
+import com.googlesource.gerrit.plugins.auditsl4j.TransformableLog;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HTTPLog implements TransformableLog {
+ private static final Logger log = LoggerFactory.getLogger(HTTPLog.class);
+ private String ip;
+ private String user;
+ private String method;
+ private String timestamp;
+ private String resource;
+ private String protocol;
+ private Integer status;
+ private String contentLength;
+ private String referrer;
+ private String userAgent;
+
+ public HTTPLog(
+ String ip,
+ String user,
+ String timestamp,
+ String method,
+ String resource,
+ String protocol,
+ Integer status,
+ String contentLength,
+ String referrer,
+ String userAgent) {
+ this.ip = ip;
+ this.user = user;
+ this.method = method;
+ this.timestamp = timestamp;
+ this.resource = resource;
+ this.protocol = protocol;
+ this.status = status;
+ this.contentLength = contentLength;
+ this.referrer = referrer;
+ this.userAgent = userAgent;
+ }
+
+ public static Optional<HTTPLog> createFromLog(String line) {
+ // HTTP log example:
+ // 104.32.164.100 - - [24/Jan/2019:00:00:03 +0000] "GET /plugins/events-log/ HTTP/1.1" 404 9 -
+ // "Apache-HttpClient/4.5.3 (Java/1.8.0_191)"
+ Matcher a =
+ Pattern.compile(
+ "^(?<ip>.*?)\\s-\\s(?<user>.*?)\\s\\["
+ + "(?<timestamp>.*?)\\]\\s\""
+ + "(?<method>\\w+)\\s(?<resource>.*?)\\s(?<protocol>.*?)\"\\s"
+ + "(?<status>\\d+)\\s(?<contentLength>\\d+|-)\\s(?<referrer>.*?)\\s(?<userAgent>.*?)$")
+ .matcher(line);
+ if (a.matches()) {
+ try {
+ return Optional.of(
+ new HTTPLog(
+ a.group("ip"),
+ a.group("user"),
+ a.group("timestamp"),
+ a.group("method"),
+ a.group("resource"),
+ a.group("protocol"),
+ Integer.parseInt(a.group("status")),
+ a.group("contentLength"),
+ a.group("referrer"),
+ a.group("userAgent")));
+ } catch (Exception e) {
+ log.error("Something wrong while parsing line: " + line);
+ }
+ } else {
+ log.error("Can't extract any info from line: " + line);
+ }
+ return Optional.empty();
+ }
+
+ private AuditUser getAuditUser() {
+ AuditUser au = new AuditUser();
+ au.setUserName(this.user);
+ au.setAccessPath(getAccessPath());
+ return au;
+ }
+
+ private AccessPath getAccessPath() {
+ Matcher a = Pattern.compile("^\"git.*?").matcher(this.userAgent);
+ return a.matches() ? AccessPath.GIT : AccessPath.REST_API;
+ }
+
+ public Optional<Long> getWhen() {
+ SimpleDateFormat format = new SimpleDateFormat("dd/MMM/yyyy:hh:mm:ss Z");
+ try {
+ return Optional.of(format.parse(this.timestamp).getTime());
+ } catch (ParseException pe) {
+ log.error(
+ "Can't parse timestamp: '" + this.timestamp + "'. Error message: " + pe.getMessage());
+ }
+ return Optional.empty();
+ }
+
+ public Optional<String> toAuditLog(LoggerAudit loggerAudit) {
+ return getWhen()
+ .map(
+ when ->
+ new HttpAuditEvent(
+ UNKNOWN_SESSION_ID,
+ this.getAuditUser(),
+ this.resource,
+ when,
+ null,
+ this.method,
+ null,
+ this.status,
+ null))
+ .map(
+ httpAuditEvent -> {
+ AuditUser au = getAuditUser();
+ if (au.getAccessPath() == AccessPath.REST_API) {
+ return loggerAudit.getAuditString(
+ httpAuditEvent, TransformableAuditLogType.ExtendedHttpAuditEvent);
+ } else {
+ return loggerAudit.getAuditString(
+ httpAuditEvent, TransformableAuditLogType.HttpAuditEvent);
+ }
+ });
+ }
+
+ public static String logFilenameBase() {
+ return "httpd_log";
+ }
+
+ public String getIp() {
+ return ip;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getMethod() {
+ return method;
+ }
+
+ public String getTimestamp() {
+ return timestamp;
+ }
+
+ public String getResource() {
+ return resource;
+ }
+
+ public String getProtocol() {
+ return protocol;
+ }
+
+ public Integer getStatus() {
+ return status;
+ }
+
+ public String getContentLength() {
+ return contentLength;
+ }
+
+ public String getReferrer() {
+ return referrer;
+ }
+
+ public String getUserAgent() {
+ return userAgent;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLog.java b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLog.java
new file mode 100644
index 0000000..03d9bed
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLog.java
@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.googlesource.gerrit.plugins.auditsl4j.logsource;
+
+import com.google.gerrit.server.AccessPath;
+import com.google.gerrit.server.audit.SshAuditEvent;
+import com.googlesource.gerrit.plugins.auditsl4j.AuditUser;
+import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit;
+import com.googlesource.gerrit.plugins.auditsl4j.TransformableAuditLogType;
+import com.googlesource.gerrit.plugins.auditsl4j.TransformableLog;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SSHLog implements TransformableLog {
+ private static final Logger log = LoggerFactory.getLogger(SSHLog.class);
+
+ private String user;
+ private String session;
+ private String timestamp;
+ private String accountId;
+ private String command;
+ private String waitTime;
+ private String execTime;
+ private String result;
+
+ public SSHLog(
+ String timestamp,
+ String session,
+ String user,
+ String accountId,
+ String command,
+ String waitTime,
+ String execTime,
+ String result) {
+ this.user = user;
+ this.session = session;
+ this.timestamp = timestamp;
+ this.accountId = accountId;
+ this.command = command;
+ this.waitTime = waitTime;
+ this.execTime = execTime;
+ this.result = result;
+ }
+
+ public static Optional<SSHLog> createFromLog(String line) {
+ Matcher authCommand =
+ Pattern.compile(
+ "^\\[(?<timestamp>.*?)\\]\\s(?<session>.*?)\\s"
+ + "(?<user>.*?)\\s(?<accountId>.*?)\\s(?<command>LOGOUT|LOGIN)(:?\\sFROM.*?)?$")
+ .matcher(line);
+
+ Matcher authFailure =
+ Pattern.compile(
+ "^\\[(?<timestamp>.*?)\\]\\s(?<session>.*?)\\s(?<user>.*?)\\s(?<command>AUTH FAILURE)(:?\\sFROM.*?)?$")
+ .matcher(line);
+
+ Matcher nonAuthCommand =
+ Pattern.compile(
+ "^\\[(?<timestamp>.*?)\\]\\s(?<session>.*?)\\s(?<user>.*?)\\s(?<accountId>.*?)\\s(?<command>.*?)(\\s(?<waitTime>\\d+ms)\\s(?<execTime>\\d+ms)\\s(?<result>.*?))?$")
+ .matcher(line);
+
+ if (authCommand.matches()) {
+ try {
+ return Optional.of(
+ new SSHLog(
+ authCommand.group("timestamp"),
+ authCommand.group("session"),
+ authCommand.group("user"),
+ authCommand.group("accountId"),
+ authCommand.group("command"),
+ null,
+ null,
+ "0"));
+ } catch (Exception e) {
+ log.error("Auth command match, but something wrong while parsing line: " + line);
+ }
+ } else if (authFailure.matches()) {
+ try {
+ return Optional.of(
+ new SSHLog(
+ authFailure.group("timestamp"),
+ authFailure.group("session"),
+ authFailure.group("user"),
+ null,
+ authFailure.group("command"),
+ null,
+ null,
+ "0"));
+ } catch (Exception e) {
+ log.error("Auth failure command match, but something wrong while parsing line: " + line);
+ }
+
+ } else if (nonAuthCommand.matches()) {
+ try {
+ return Optional.of(
+ new SSHLog(
+ nonAuthCommand.group("timestamp"),
+ nonAuthCommand.group("session"),
+ nonAuthCommand.group("user"),
+ nonAuthCommand.group("accountId"),
+ nonAuthCommand.group("command"),
+ nonAuthCommand.group("waitTime"),
+ nonAuthCommand.group("execTime"),
+ nonAuthCommand.group("result") != null ? nonAuthCommand.group("result") : "0"));
+ } catch (Exception e) {
+ log.error("Non Auth command match, but something wrong while parsing line: " + line);
+ }
+ } else {
+ log.error("Can't extract any info from line: " + line);
+ }
+ return Optional.empty();
+ }
+
+ private AuditUser getAuditUser() {
+ AuditUser au = new AuditUser();
+ au.setUserName(this.user);
+ au.setAccessPath(AccessPath.SSH_COMMAND);
+ return au;
+ }
+
+ private Optional<Long> getWhen() {
+ // Timestamp format example: 2019-01-23 12:44:04,723 +0100
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS Z");
+ try {
+ return Optional.of(format.parse(this.timestamp).getTime());
+ } catch (ParseException pe) {
+ log.error(
+ "Can't parse timestamp: '" + this.timestamp + "'. Error message: " + pe.getMessage());
+ }
+ return Optional.empty();
+ }
+
+ public Optional<String> toAuditLog(LoggerAudit loggerAudit) {
+ return getWhen()
+ .map(
+ when ->
+ new SshAuditEvent(
+ this.session, getAuditUser(), this.command, when, null, this.result))
+ .map(
+ sshAuditEvent ->
+ loggerAudit.getAuditString(sshAuditEvent, TransformableAuditLogType.SshAuditEvent));
+ }
+
+ public static String logFilenameBase() {
+ return "sshd_log";
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getSession() {
+ return session;
+ }
+
+ public String getTimestamp() {
+ return timestamp;
+ }
+
+ public String getAccountId() {
+ return accountId;
+ }
+
+ public String getCommand() {
+ return command;
+ }
+
+ public String getWaitTime() {
+ return waitTime;
+ }
+
+ public String getExecTime() {
+ return execTime;
+ }
+
+ public String getResult() {
+ return result;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HttpLogTest.java b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HttpLogTest.java
new file mode 100644
index 0000000..756c5cb
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/HttpLogTest.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.googlesource.gerrit.plugins.auditsl4j.logsource;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.googlesource.gerrit.plugins.auditsl4j.AuditRendererToJson;
+import com.googlesource.gerrit.plugins.auditsl4j.AuditWriterToStringList;
+import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit;
+import java.util.Optional;
+import org.junit.Test;
+
+public class HttpLogTest {
+
+ LoggerAudit loggerAudit =
+ new LoggerAudit(new AuditWriterToStringList(), new AuditRendererToJson());
+
+ @Test
+ public void succesullyParseHttpLog() {
+ String ip = "104.32.164.100";
+ String user = "myUser";
+ String method = "GET";
+ String timestamp = "24/Jan/2019:00:00:03 +0000";
+ String resource = "/plugins/events-log/";
+ String protocol = "HTTP/1.1";
+ Integer status = 404;
+ String contentLenght = "9";
+ String referrer = "ciccio";
+ String userAgent = "\"Apache-HttpClient/4.5.3 (Java/1.8.0_191)\"";
+ // 104.32.164.100 - - [24/Jan/2019:00:00:03 +0000] "GET /plugins/events-log/ HTTP/1.1" 404 9 -
+ // "Apache-HttpClient/4.5.3 (Java/1.8.0_191)"
+ String logLine =
+ String.format(
+ "%s - %s [%s] \"%s %s %s\" %d %s %s %s",
+ ip,
+ user,
+ timestamp,
+ method,
+ resource,
+ protocol,
+ status,
+ contentLenght,
+ referrer,
+ userAgent);
+
+ HTTPLog expected =
+ new HTTPLog(
+ ip,
+ user,
+ timestamp,
+ method,
+ resource,
+ protocol,
+ status,
+ contentLenght,
+ referrer,
+ userAgent);
+ Optional<HTTPLog> maybeHTTPLog = HTTPLog.createFromLog(logLine);
+ assertTrue("Didn't create HTTPLog", maybeHTTPLog.isPresent());
+ HTTPLog gotHTTPLog = maybeHTTPLog.get();
+ assertEquals(expected.getIp(), gotHTTPLog.getIp());
+ assertEquals(expected.getUser(), gotHTTPLog.getUser());
+ assertEquals(expected.getMethod(), gotHTTPLog.getMethod());
+ assertEquals(expected.getTimestamp(), gotHTTPLog.getTimestamp());
+ assertEquals(expected.getResource(), gotHTTPLog.getResource());
+ assertEquals(expected.getReferrer(), gotHTTPLog.getReferrer());
+ assertEquals(expected.getProtocol(), gotHTTPLog.getProtocol());
+ assertEquals(expected.getStatus(), gotHTTPLog.getStatus());
+ assertEquals(expected.getContentLength(), gotHTTPLog.getContentLength());
+ assertEquals(expected.getUserAgent(), gotHTTPLog.getUserAgent());
+ }
+
+ @Test
+ public void handleIPV6ParseHttpLog() {
+ String logLine =
+ "2405:204:a313:675::17a:b0b1 - - [19/Jan/2019:18:15:02 +0000] \"GET /plugins/codemirror-editor/static/codemirror_editor.js HTTP/1.1\" 200 1498 \"https://gerrithub.io/plugins/codemirror-editor/static/codemirror_editor.html\" \"Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko; googleweblight) Chrome/38.0.1025.166 Mobile Safari/535.19\"";
+ Optional<HTTPLog> maybeHTTPLog = HTTPLog.createFromLog(logLine);
+ assertTrue("Didn't create HTTPLog", maybeHTTPLog.isPresent());
+ assertEquals("2405:204:a313:675::17a:b0b1", maybeHTTPLog.get().getIp());
+ }
+
+ @Test(expected = Test.None.class /* no exception expected */)
+ public void handleNonNumericalContentLengthParseHttpLog() {
+ String logLine =
+ "171.13.14.52 - - [19/Jan/2019:00:00:46 +0000] \"HEAD /Documentation/index.html HTTP/1.1\" 200 - - \"Mozilla/5.0 (Windows NT 10.0 WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36\"";
+ HTTPLog.createFromLog(logLine);
+ }
+
+ @Test
+ public void invalidHttpLog() {
+ assertEquals(Optional.empty(), HTTPLog.createFromLog("invalid http log"));
+ }
+
+ @Test(expected = Test.None.class /* no exception expected */)
+ public void correctlyTransform_HttpAuditEvent() throws Exception {
+ HTTPLog auditHTTPLog =
+ new HTTPLog(
+ "10.10.100.108",
+ "anyUser",
+ "19/Jan/2019:00:00:00 +0000",
+ "amnyResource",
+ "anyProtocol",
+ "-",
+ 200,
+ "-",
+ "-",
+ "\"git/1.8.3.1\"");
+
+ Optional<String> maybeAuditLog = auditHTTPLog.toAuditLog(loggerAudit);
+ assertTrue("Didn't Audit Log", maybeAuditLog.isPresent());
+
+ String auditLog = maybeAuditLog.get();
+ assertTrue(auditLog.contains("\"type\":\"HttpAuditEvent\""));
+ assertTrue(auditLog.contains("\"access_path\":\"GIT\""));
+ }
+
+ @Test(expected = Test.None.class /* no exception expected */)
+ public void correctlyTransform_ExtendedHttpAuditEvent() throws Exception {
+ HTTPLog auditHTTPLog =
+ new HTTPLog(
+ "10.10.100.108",
+ "anyUser",
+ "19/Jan/2019:00:00:00 +0000",
+ "amnyResource",
+ "anyProtocol",
+ "-",
+ 200,
+ "-",
+ "-",
+ "\"anyUserAgent\"");
+
+ Optional<String> maybeAuditLog = auditHTTPLog.toAuditLog(loggerAudit);
+ assertTrue("Didn't Audit Log", maybeAuditLog.isPresent());
+
+ String auditLog = maybeAuditLog.get();
+ assertTrue(auditLog.contains("\"type\":\"ExtendedHttpAuditEvent\""));
+ assertTrue(auditLog.contains("\"access_path\":\"REST_API\""));
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLogTest.java b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLogTest.java
new file mode 100644
index 0000000..835104f
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/auditsl4j/logsource/SSHLogTest.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.googlesource.gerrit.plugins.auditsl4j.logsource;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.googlesource.gerrit.plugins.auditsl4j.AuditRendererToJson;
+import com.googlesource.gerrit.plugins.auditsl4j.AuditWriterToStringList;
+import com.googlesource.gerrit.plugins.auditsl4j.LoggerAudit;
+import java.util.Optional;
+import org.junit.Test;
+
+public class SSHLogTest {
+
+ LoggerAudit loggerAudit =
+ new LoggerAudit(new AuditWriterToStringList(), new AuditRendererToJson());
+
+ @Test
+ public void successfullyParseSshLog() {
+ String user = "myUser";
+ String timestamp = "2019-01-23 12:43:53,115 +0100";
+ String session = "b015fbe2";
+ String accountId = "a/1000000";
+ String command = "audit-sl4j.import.--from.rewew.--until.sdfds";
+ String waitTime = "2ms";
+ String execTime = "2ms";
+ String result = "FAIL";
+
+ // [2019-01-23 12:44:04,723 +0100] b015fbe2 admin a/1000000
+ // audit-sl4j.import.--from.rewew.--until.sdfds 2ms 2ms 0
+ String logLine =
+ String.format(
+ "[%s] %s %s %s %s %s %s %s",
+ timestamp, session, user, accountId, command, waitTime, execTime, result);
+
+ SSHLog expected =
+ new SSHLog(timestamp, session, user, accountId, command, waitTime, execTime, result);
+
+ Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine);
+ assertTrue("Didn't create SSHLog", maybeSshLog.isPresent());
+ SSHLog gotSSHLog = maybeSshLog.get();
+ assertEquals(expected.getTimestamp(), gotSSHLog.getTimestamp());
+ assertEquals(expected.getSession(), gotSSHLog.getSession());
+ assertEquals(expected.getUser(), gotSSHLog.getUser());
+ assertEquals(expected.getAccountId(), gotSSHLog.getAccountId());
+ assertEquals(expected.getCommand(), gotSSHLog.getCommand());
+ assertEquals(expected.getWaitTime(), gotSSHLog.getWaitTime());
+ assertEquals(expected.getExecTime(), gotSSHLog.getExecTime());
+ assertEquals(expected.getResult(), gotSSHLog.getResult());
+ }
+
+ @Test
+ public void successfullyParseSshLogWithParameters() {
+ String logLine =
+ "[2019-01-20 00:20:24,277 +0000] e4e82e5e spdk-bot a/1011203 gerrit.query.--format.json.--current-patch-set.status:open project:spdk/spdk.github.io label:Verified=0 NOT is:draft 1ms 1ms 0";
+ Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine);
+ assertTrue("Didn't create SSHLog", maybeSshLog.isPresent());
+ assertEquals(
+ "gerrit.query.--format.json.--current-patch-set.status:open project:spdk/spdk.github.io label:Verified=0 NOT is:draft",
+ maybeSshLog.get().getCommand());
+ assertEquals("0", maybeSshLog.get().getResult());
+ }
+
+ @Test
+ public void handleLogout() {
+ String logLine = "[2019-01-23 12:44:26,665 +0100] 70e3031f admin a/1000000 LOGOUT";
+ Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine);
+ assertTrue("Didn't create SSHLog", maybeSshLog.isPresent());
+ assertEquals("LOGOUT", maybeSshLog.get().getCommand());
+ assertEquals("0", maybeSshLog.get().getResult());
+ }
+
+ @Test
+ public void handleLogin() {
+ String logLine =
+ "[2019-01-01 00:00:05,613 +0000] e126989b spdk-bot a/1011203 LOGIN FROM 172.19.0.1";
+ Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine);
+ assertTrue("Didn't create SSHLog", maybeSshLog.isPresent());
+ assertEquals("LOGIN", maybeSshLog.get().getCommand());
+ assertEquals("0", maybeSshLog.get().getResult());
+ }
+
+ @Test
+ public void handleNonAuthCommandWithoutTiming() {
+ String logLine =
+ "[2018-09-22 15:44:30,539 +0000] ce2a2263 vogella-jenkins a/1012807 gerrit.review.426428,1.--message.Build Failed";
+ Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine);
+ assertTrue("Didn't create SSHLog", maybeSshLog.isPresent());
+ assertEquals("gerrit.review.426428,1.--message.Build Failed", maybeSshLog.get().getCommand());
+ assertEquals("0", maybeSshLog.get().getResult());
+ }
+
+ @Test
+ public void handleCommandWithMultiSpaces() {
+ String logLine =
+ "[2018-09-22 16:28:30,668 +0000] 61e0f7b9 vogella-jenkins a/1012807 gerrit.review.426428,2.--message.Build Started http://build.vogella.com:8080/job/learn.vogella.com-Gerrit/1323/ .--verified.0.--code-review.0 1ms 303ms 0";
+ Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine);
+ assertTrue("Didn't create SSHLog", maybeSshLog.isPresent());
+ assertEquals(
+ "gerrit.review.426428,2.--message.Build Started http://build.vogella.com:8080/job/learn.vogella.com-Gerrit/1323/ .--verified.0.--code-review.0",
+ maybeSshLog.get().getCommand());
+ assertEquals("0", maybeSshLog.get().getResult());
+ }
+
+ @Test
+ public void handleAuthFailure() {
+ String logLine =
+ "[2018-09-03 18:14:43,831 +0000] f540bf46 - AUTH FAILURE FROM 172.19.0.1 user-not-found";
+ Optional<SSHLog> maybeSshLog = SSHLog.createFromLog(logLine);
+ assertTrue("Didn't create SSHLog", maybeSshLog.isPresent());
+ assertEquals("AUTH FAILURE", maybeSshLog.get().getCommand());
+ assertEquals("0", maybeSshLog.get().getResult());
+ }
+
+ @Test(expected = Test.None.class /* no exception expected */)
+ public void correctlyTransform_SshAuditEvent() throws Exception {
+ SSHLog SSHLog =
+ new SSHLog(
+ "2019-01-23 12:44:26,665 +0100",
+ "70e3031f",
+ "admin",
+ "a/1000000",
+ "LOGOUT",
+ "2ms",
+ "2ms",
+ "0");
+
+ Optional<String> maybeAuditLog = SSHLog.toAuditLog(loggerAudit);
+ assertTrue("Didn't Audit Log", maybeAuditLog.isPresent());
+
+ String auditLog = maybeAuditLog.get();
+ assertTrue("'type' not matched: " + auditLog, auditLog.contains("\"type\":\"SshAuditEvent\""));
+ assertTrue(
+ "'access_path' not matched: " + auditLog,
+ auditLog.contains("\"access_path\":\"SSH_COMMAND\""));
+ assertTrue("'when' not matched: " + auditLog, auditLog.contains("\"when\":1548243866665"));
+ }
+}
