Stream rendered Markdown to browser
Instead of buffering the fully formatted HTML for the entire page,
stream the content as it is being produced from the AST. This reduces
the memory footprint of the Gitiles server while rendering large
documents, and can reduce latency to display content above the fold.
Change-Id: Ic57d1c0f6fd8daf31f82f3ced0536cd1e3030fa0
diff --git a/gitiles-servlet/src/main/java/com/google/gitiles/BaseServlet.java b/gitiles-servlet/src/main/java/com/google/gitiles/BaseServlet.java
index 874e593..387a15f 100644
--- a/gitiles-servlet/src/main/java/com/google/gitiles/BaseServlet.java
+++ b/gitiles-servlet/src/main/java/com/google/gitiles/BaseServlet.java
@@ -280,7 +280,7 @@
if (!allData.containsKey("repositoryName") && view.getRepositoryName() != null) {
allData.put("repositoryName", view.getRepositoryName());
}
- if (!allData.containsKey("breadcrumbs")) {
+ if (!allData.containsKey("breadcrumbs") && view.getRepositoryName() != null) {
allData.put("breadcrumbs", view.getBreadcrumbs());
}
diff --git a/gitiles-servlet/src/main/java/com/google/gitiles/doc/DocServlet.java b/gitiles-servlet/src/main/java/com/google/gitiles/doc/DocServlet.java
index 3c7f54a..bd5c481 100644
--- a/gitiles-servlet/src/main/java/com/google/gitiles/doc/DocServlet.java
+++ b/gitiles-servlet/src/main/java/com/google/gitiles/doc/DocServlet.java
@@ -14,7 +14,6 @@
package com.google.gitiles.doc;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND;
import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
@@ -25,16 +24,19 @@
import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
+import com.google.common.base.Throwables;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.google.common.net.HttpHeaders;
import com.google.gitiles.BaseServlet;
-import com.google.gitiles.FormatType;
import com.google.gitiles.GitilesAccess;
import com.google.gitiles.GitilesView;
import com.google.gitiles.Renderer;
import com.google.gitiles.ViewFilter;
+import com.google.gitiles.doc.html.StreamHtmlBuilder;
import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nullable;
@@ -165,14 +167,9 @@
MarkdownFile srcFile)
throws IOException {
Map<String, Object> data = new HashMap<>();
- Navbar navbar = new Navbar();
- if (navFile != null) {
- navbar.setFormatter(fmt.setFilePath(navFile.path).build());
- navbar.setMarkdown(navFile.content);
- }
- data.putAll(navbar.toSoyData());
+ data.putAll(buildNavbar(fmt, navFile));
- Node doc = GitilesMarkdown.parse(srcFile.content);
+ Node doc = GitilesMarkdown.parse(srcFile.consumeContent());
data.put("pageTitle", pageTitle(doc, srcFile));
if (view.getType() != GitilesView.Type.ROOTED_DOC) {
data.put("sourceUrl", GitilesView.show().copyFrom(view).toUrl());
@@ -182,21 +179,24 @@
if (cfg.analyticsId != null) {
data.put("analyticsId", cfg.analyticsId);
}
- data.put("bodyHtml", fmt.setFilePath(srcFile.path).build().toSoyHtml(doc));
- String page = renderer.render(SOY_TEMPLATE, data);
- byte[] raw = page.getBytes(UTF_8);
- res.setContentType(FormatType.HTML.getMimeType());
- res.setCharacterEncoding(UTF_8.name());
- setCacheHeaders(req, res);
- if (acceptsGzipEncoding(req)) {
- res.addHeader(HttpHeaders.VARY, HttpHeaders.ACCEPT_ENCODING);
- res.setHeader(HttpHeaders.CONTENT_ENCODING, "gzip");
- raw = gzip(raw);
+ try (OutputStream out = startRenderCompressedStreamingHtml(req, res, SOY_TEMPLATE, data)) {
+ Writer w = newWriter(out, res);
+ fmt.setFilePath(srcFile.path).build().renderToHtml(new StreamHtmlBuilder(w), doc);
+ w.flush();
+ } catch (RuntimeIOException e) {
+ Throwables.throwIfInstanceOf(e.getCause(), IOException.class);
+ throw e;
}
- res.setContentLength(raw.length);
- res.setStatus(HttpServletResponse.SC_OK);
- res.getOutputStream().write(raw);
+ }
+
+ private Map<String, Object> buildNavbar(MarkdownToHtml.Builder fmt, MarkdownFile navFile) {
+ Navbar navbar = new Navbar();
+ if (navFile != null) {
+ navbar.setFormatter(fmt.setFilePath(navFile.path).build());
+ navbar.setMarkdown(navFile.consumeContent());
+ }
+ return navbar.toSoyData();
}
private static String pageTitle(Node doc, MarkdownFile srcFile) {
@@ -282,5 +282,11 @@
void read(ObjectReader reader, MarkdownConfig cfg) throws IOException {
content = reader.open(id, OBJ_BLOB).getCachedBytes(cfg.inputLimit);
}
+
+ byte[] consumeContent() {
+ byte[] c = content;
+ content = null;
+ return c;
+ }
}
}
diff --git a/gitiles-servlet/src/main/java/com/google/gitiles/doc/MarkdownToHtml.java b/gitiles-servlet/src/main/java/com/google/gitiles/doc/MarkdownToHtml.java
index 99f1c53..56199b9 100644
--- a/gitiles-servlet/src/main/java/com/google/gitiles/doc/MarkdownToHtml.java
+++ b/gitiles-servlet/src/main/java/com/google/gitiles/doc/MarkdownToHtml.java
@@ -21,6 +21,7 @@
import com.google.gitiles.GitilesView;
import com.google.gitiles.ThreadSafePrettifyParser;
import com.google.gitiles.doc.html.HtmlBuilder;
+import com.google.gitiles.doc.html.SoyHtmlBuilder;
import com.google.template.soy.data.SanitizedContent;
import com.google.template.soy.shared.restricted.EscapingConventions.FilterImageDataUri;
import com.google.template.soy.shared.restricted.EscapingConventions.FilterNormalizeUri;
@@ -118,8 +119,8 @@
}
}
- private final HtmlBuilder html = new HtmlBuilder();
- private final TocFormatter toc = new TocFormatter(html, 3);
+ private HtmlBuilder html;
+ private TocFormatter toc;
private final String requestUri;
private final GitilesView view;
private final MarkdownConfig config;
@@ -143,14 +144,26 @@
}
/** Render the document AST to sanitized HTML. */
- public SanitizedContent toSoyHtml(Node node) {
- if (node == null) {
- return null;
+ public void renderToHtml(HtmlBuilder out, Node node) {
+ if (node != null) {
+ html = out;
+ toc = new TocFormatter(html, 3);
+ toc.setRoot(node);
+ node.accept(this);
+ html.finish();
+ html = null;
+ toc = null;
}
+ }
- toc.setRoot(node);
- node.accept(this);
- return html.toSoy();
+ /** Render the document AST to sanitized HTML. */
+ public SanitizedContent toSoyHtml(Node node) {
+ if (node != null) {
+ SoyHtmlBuilder out = new SoyHtmlBuilder();
+ renderToHtml(out, node);
+ return out.toSoy();
+ }
+ return null;
}
@Override
diff --git a/gitiles-servlet/src/main/java/com/google/gitiles/doc/RuntimeIOException.java b/gitiles-servlet/src/main/java/com/google/gitiles/doc/RuntimeIOException.java
new file mode 100644
index 0000000..8ef7923
--- /dev/null
+++ b/gitiles-servlet/src/main/java/com/google/gitiles/doc/RuntimeIOException.java
@@ -0,0 +1,26 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gitiles.doc;
+
+import java.io.IOException;
+
+/** {@link IOException} wrapped inside RuntimeException. */
+public class RuntimeIOException extends RuntimeException {
+ private static final long serialVersionUID = 1L;
+
+ public RuntimeIOException(IOException cause) {
+ super(cause);
+ }
+}
diff --git a/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/HtmlBuilder.java b/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/HtmlBuilder.java
index 9551e94..b8f43b0 100644
--- a/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/HtmlBuilder.java
+++ b/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/HtmlBuilder.java
@@ -19,9 +19,7 @@
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
-import com.google.template.soy.data.SanitizedContent;
-import com.google.template.soy.data.SanitizedContent.ContentKind;
-import com.google.template.soy.data.UnsafeSanitizedContentOrdainer;
+import com.google.gitiles.doc.RuntimeIOException;
import com.google.template.soy.shared.restricted.EscapingConventions.EscapeHtml;
import com.google.template.soy.shared.restricted.EscapingConventions.FilterImageDataUri;
import com.google.template.soy.shared.restricted.EscapingConventions.FilterNormalizeUri;
@@ -37,8 +35,10 @@
* <p>Useful but critical attributes like {@code href} on anchors or {@code src} on img permit only
* safe subset of URIs, primarily {@code http://}, {@code https://}, and for image src {@code
* data:image/*;base64,...}.
+ *
+ * <p>See concrete subclasses {@link SoyHtmlBuilder} and {@link StreamHtmlBuilder}.
*/
-public final class HtmlBuilder {
+public abstract class HtmlBuilder {
private static final ImmutableSet<String> ALLOWED_TAGS =
ImmutableSet.of(
"h1",
@@ -112,12 +112,12 @@
return GIT_URI.matcher(val).find();
}
- private final StringBuilder htmlBuf;
+ private final Appendable htmlBuf;
private final Appendable textBuf;
private String tag;
- public HtmlBuilder() {
- htmlBuf = new StringBuilder();
+ HtmlBuilder(Appendable out) {
+ htmlBuf = out;
textBuf = EscapeHtml.INSTANCE.escape(htmlBuf);
}
@@ -125,7 +125,11 @@
public HtmlBuilder open(String tagName) {
checkArgument(ALLOWED_TAGS.contains(tagName), "invalid HTML tag %s", tagName);
finishActiveTag();
- htmlBuf.append('<').append(tagName);
+ try {
+ htmlBuf.append('<').append(tagName);
+ } catch (IOException e) {
+ throw new RuntimeIOException(e);
+ }
tag = tagName;
return this;
}
@@ -167,7 +171,7 @@
htmlBuf.append('"');
return this;
} catch (IOException e) {
- throw new IllegalStateException(e);
+ throw new RuntimeIOException(e);
}
}
@@ -190,10 +194,14 @@
private void finishActiveTag() {
if (tag != null) {
- if (SELF_CLOSING_TAGS.contains(tag)) {
- htmlBuf.append(" />");
- } else {
- htmlBuf.append('>');
+ try {
+ if (SELF_CLOSING_TAGS.contains(tag)) {
+ htmlBuf.append(" />");
+ } else {
+ htmlBuf.append('>');
+ }
+ } catch (IOException e) {
+ throw new RuntimeIOException(e);
}
tag = null;
}
@@ -205,7 +213,11 @@
ALLOWED_TAGS.contains(tag) && !SELF_CLOSING_TAGS.contains(tag), "invalid HTML tag %s", tag);
finishActiveTag();
- htmlBuf.append("</").append(tag).append('>');
+ try {
+ htmlBuf.append("</").append(tag).append('>');
+ } catch (IOException e) {
+ throw new RuntimeIOException(e);
+ }
return this;
}
@@ -216,14 +228,18 @@
textBuf.append(in);
return this;
} catch (IOException e) {
- throw new IllegalStateException(e);
+ throw new RuntimeIOException(e);
}
}
/** Append a space outside of an element. */
public HtmlBuilder space() {
finishActiveTag();
- htmlBuf.append(' ');
+ try {
+ htmlBuf.append(' ');
+ } catch (IOException e) {
+ throw new RuntimeIOException(e);
+ }
return this;
}
@@ -233,12 +249,15 @@
public void entity(String entity) {
checkArgument(HTML_ENTITY.matcher(entity).matches(), "invalid entity %s", entity);
finishActiveTag();
- htmlBuf.append(entity);
+ try {
+ htmlBuf.append(entity);
+ } catch (IOException e) {
+ throw new RuntimeIOException(e);
+ }
}
- /** Bless the current content as HTML. */
- public SanitizedContent toSoy() {
+ /** Finish the document. */
+ public void finish() {
finishActiveTag();
- return UnsafeSanitizedContentOrdainer.ordainAsSafe(htmlBuf.toString(), ContentKind.HTML);
}
}
diff --git a/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/SoyHtmlBuilder.java b/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/SoyHtmlBuilder.java
new file mode 100644
index 0000000..23e6ee6
--- /dev/null
+++ b/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/SoyHtmlBuilder.java
@@ -0,0 +1,39 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gitiles.doc.html;
+
+import com.google.template.soy.data.SanitizedContent;
+import com.google.template.soy.data.SanitizedContent.ContentKind;
+import com.google.template.soy.data.UnsafeSanitizedContentOrdainer;
+
+/** Builds a document fragment using a restricted subset of HTML. */
+public final class SoyHtmlBuilder extends HtmlBuilder {
+ private final StringBuilder buf;
+
+ public SoyHtmlBuilder() {
+ this(new StringBuilder());
+ }
+
+ private SoyHtmlBuilder(StringBuilder buf) {
+ super(buf);
+ this.buf = buf;
+ }
+
+ /** Bless the current content as HTML. */
+ public SanitizedContent toSoy() {
+ finish();
+ return UnsafeSanitizedContentOrdainer.ordainAsSafe(buf.toString(), ContentKind.HTML);
+ }
+}
diff --git a/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/StreamHtmlBuilder.java b/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/StreamHtmlBuilder.java
new file mode 100644
index 0000000..467dd43
--- /dev/null
+++ b/gitiles-servlet/src/main/java/com/google/gitiles/doc/html/StreamHtmlBuilder.java
@@ -0,0 +1,24 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gitiles.doc.html;
+
+import java.io.Writer;
+
+/** Writes sanitized HTML to a stream. */
+public final class StreamHtmlBuilder extends HtmlBuilder {
+ public StreamHtmlBuilder(Writer out) {
+ super(out);
+ }
+}
diff --git a/gitiles-servlet/src/main/resources/com/google/gitiles/templates/Doc.soy b/gitiles-servlet/src/main/resources/com/google/gitiles/templates/Doc.soy
index 2193c93..e43d2c1 100644
--- a/gitiles-servlet/src/main/resources/com/google/gitiles/templates/Doc.soy
+++ b/gitiles-servlet/src/main/resources/com/google/gitiles/templates/Doc.soy
@@ -25,7 +25,6 @@
* @param logUrl url for log history of page.
* @param blameUrl url for blame of page source.
* @param? navbarHtml markdown ast node to convert.
- * @param bodyHtml safe html to embed into the body of the page.
*/
{template .markdownDoc}
<!DOCTYPE html>
@@ -59,7 +58,7 @@
<div class="Site-content Site-Content--markdown">
<div class="Container">
<div class="doc">
- {$bodyHtml}
+ {call .streamingPlaceholder /}
</div>
</div>
</div>
