Do all asciidoctor rendering in a single jvm
Previously for every html rendered, there's a separate jvm used, and buck's
parallel building will execute multiple jvm at once, burning the CPU and being
very slow.
Modified the asciidoctor-java-integrator CLI java wrapper interface to do that
all in a single jvm.
On a workstation tested, "buck build Documentation:html" took about 140s before,
and 60s after.
As a side-effect, we lose the build rules of the single html files (e.g.
"buck build Documentation:licenses.html")
Change-Id: Ifc70c63676b59571c6e240636752b7cba9270f04
diff --git a/Documentation/BUCK b/Documentation/BUCK
index faa2553..2876f13 100644
--- a/Documentation/BUCK
+++ b/Documentation/BUCK
@@ -4,48 +4,41 @@
MAIN = ['//gerrit-pgm:pgm', '//gerrit-gwtui:ui_module']
SRCS = glob(['*.txt'], excludes = ['licenses.txt'])
-HTML = [txt[0:-4] + '.html' for txt in SRCS]
genrule(
name = 'html',
cmd = 'cd $TMP;' +
'mkdir -p Documentation/images;' +
+ 'unzip -q $SRCDIR/only_html.zip -d Documentation/;' +
'for s in $SRCS;do ln -s $s Documentation;done;' +
'mv Documentation/*.{jpg,png} Documentation/images;' +
+ 'rm Documentation/only_html.zip;' +
'rm Documentation/licenses.txt;' +
'cp $SRCDIR/licenses.txt LICENSES.txt;' +
'zip -qr $OUT *',
- srcs = [genfile(d) for d in HTML] +
+ srcs = [genfile('only_html.zip')] +
glob([
'images/*.jpg',
'images/*.png',
]) + [
- genfile('doc.css'),
- genfile('licenses.html'),
+ 'doc.css',
genfile('licenses.txt'),
],
- deps = [':' + d for d in HTML] + [
- ':licenses.html',
+ deps = [
+ ':generate_html',
':licenses.txt',
- ':doc.css',
],
out = 'html.zip',
visibility = ['PUBLIC'],
)
-genrule(
- name = 'doc.css',
- cmd = 'ln -s $SRCDIR/doc.css $OUT',
- srcs = ['doc.css'],
- out = 'doc.css',
-)
-
genasciidoc(
+ name = 'generate_html',
srcs = SRCS + [genfile('licenses.txt')],
- outs = HTML + ['licenses.html'],
- deps = DOCUMENTATION_DEPS,
+ deps = [':licenses.txt'],
attributes = documentation_attributes(git_describe()),
backend = 'html5',
+ out = 'only_html.zip',
)
genrule(
diff --git a/Documentation/asciidoc.defs b/Documentation/asciidoc.defs
index 8279847..e2de785 100644
--- a/Documentation/asciidoc.defs
+++ b/Documentation/asciidoc.defs
@@ -13,25 +13,30 @@
# limitations under the License.
def genasciidoc(
+ name,
+ out,
srcs = [],
- outs = [],
- deps = {},
+ deps = [],
attributes = [],
backend = None,
visibility = []):
EXPN = '.expn'
- asciidoc = ['$(exe //lib/asciidoctor:asciidoc)']
+ asciidoc = [
+ '$(exe //lib/asciidoctor:asciidoc)',
+ '-z', '$OUT',
+ '--in-ext', '".txt%s"' % EXPN,
+ '--out-ext', '".html"',
+ ]
if backend:
asciidoc.extend(['-b', backend])
for attribute in attributes:
asciidoc.extend(['-a', attribute])
- asciidoc.extend(['-o', '$OUT'])
+ asciidoc.append('$SRCS')
+ newsrcs = []
+ newdeps = deps + ['//lib/asciidoctor:asciidoc']
- for p in zip(srcs, outs):
- src, out = p
- dep = deps.get(src) or []
-
+ for src in srcs:
tx = []
fn = src
if fn.startswith('BUCKGEN:') :
@@ -48,14 +53,14 @@
deps = tx + [':replace_macros'],
out = ex,
)
- genrule(
- name = out,
- cmd = ' '.join(asciidoc + ['$SRCDIR/' + ex]),
- srcs = [genfile(ex)] + [genfile(n + EXPN) for n in dep],
- deps = [':' + n + EXPN for n in dep] + [
- ':' + ex,
- '//lib/asciidoctor:asciidoc',
- ],
- out = out,
- visibility = visibility,
- )
+ newdeps.append(':' + ex)
+ newsrcs.append(genfile(ex))
+
+ genrule(
+ name = name,
+ cmd = ' '.join(asciidoc),
+ srcs = newsrcs,
+ deps = newdeps,
+ out = out,
+ visibility = visibility,
+ )
diff --git a/lib/asciidoctor/BUCK b/lib/asciidoctor/BUCK
index 25b9eb8..6a1fdd7 100644
--- a/lib/asciidoctor/BUCK
+++ b/lib/asciidoctor/BUCK
@@ -2,11 +2,21 @@
java_binary(
name = 'asciidoc',
- main_class = 'org.asciidoctor.cli.AsciidoctorInvoker',
- deps = [':asciidoctor'],
+ main_class = 'Main',
+ deps = [':main_lib'],
visibility = ['PUBLIC'],
)
+java_library(
+ name = 'main_lib',
+ srcs = ['java/Main.java'],
+ deps = [
+ ':asciidoctor',
+ ':jruby',
+ '//lib:args4j',
+ ],
+)
+
maven_jar(
name = 'asciidoctor',
id = 'org.asciidoctor:asciidoctor-java-integration:0.1.3',
@@ -14,19 +24,6 @@
license = 'Apache2.0',
visibility = [],
attach_source = False,
- deps = [
- ':jcommander',
- ':jruby',
- ],
-)
-
-maven_jar(
- name = 'jcommander',
- id = 'com.beust:jcommander:1.30',
- sha1 = 'c440b30a944ba199751551aee393f8aa03b3c327',
- license = 'Apache2.0',
- visibility = [],
- attach_source = False,
)
maven_jar(
diff --git a/lib/asciidoctor/java/Main.java b/lib/asciidoctor/java/Main.java
new file mode 100644
index 0000000..eb1ef33
--- /dev/null
+++ b/lib/asciidoctor/java/Main.java
@@ -0,0 +1,159 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.asciidoctor.Asciidoctor;
+import org.asciidoctor.AttributesBuilder;
+import org.asciidoctor.Options;
+import org.asciidoctor.OptionsBuilder;
+import org.asciidoctor.internal.JRubyAsciidoctor;
+
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+public class Main {
+
+ private static final int BUFSIZ = 4096;
+ private static final String DOCTYPE = "article";
+ private static final String ERUBY = "erb";
+
+ @Option(name = "-b", usage = "set output format backend")
+ private String backend = "html5";
+
+ @Option(name = "-z", usage = "output zip file")
+ private String zipFile;
+
+ @Option(name = "--in-ext", usage = "extension for input files")
+ private String inExt = ".txt";
+
+ @Option(name = "--out-ext", usage = "extension for output files")
+ private String outExt = ".html";
+
+ @Option(name = "-a", usage =
+ "a list of attributes, in the form key or key=value pair")
+ private List<String> attributes = new ArrayList<String>();
+
+ @Argument(usage = "input files")
+ private List<String> inputFiles = new ArrayList<String>();
+
+ private String mapInFileToOutFile(String inFile) {
+ String basename = new File(inFile).getName();
+ if (basename.endsWith(inExt)) {
+ basename = basename.substring(0, basename.length() - inExt.length());
+ } else {
+ // Strip out the last extension
+ int pos = basename.lastIndexOf('.');
+ if (pos > 0) {
+ basename = basename.substring(0, pos);
+ }
+ }
+ return basename + outExt;
+ }
+
+ private Options createOptions(File tmpFile) {
+ OptionsBuilder optionsBuilder = OptionsBuilder.options();
+
+ optionsBuilder.backend(backend).docType(DOCTYPE).eruby(ERUBY);
+ // XXX(fishywang): ideally we should just output to a string and add the
+ // content into zip. But asciidoctor will actually ignore all attributes if
+ // not output to a file. So we *have* to output to a file then read the
+ // content of the file into zip.
+ optionsBuilder.toFile(tmpFile);
+
+ AttributesBuilder attributesBuilder = AttributesBuilder.attributes();
+ attributesBuilder.attributes(getAttributes());
+ optionsBuilder.attributes(attributesBuilder.get());
+
+ return optionsBuilder.get();
+ }
+
+ private Map<String, Object> getAttributes() {
+ Map<String, Object> attributeValues = new HashMap<String, Object>();
+
+ for (String attribute : attributes) {
+ int equalsIndex = attribute.indexOf('=');
+ if(equalsIndex > -1) {
+ String name = attribute.substring(0, equalsIndex);
+ String value = attribute.substring(equalsIndex + 1, attribute.length());
+
+ attributeValues.put(name, value);
+ } else {
+ attributeValues.put(attribute, "");
+ }
+ }
+
+ return attributeValues;
+ }
+
+ private void invoke(String... parameters) throws IOException {
+ CmdLineParser parser = new CmdLineParser(this);
+ try {
+ parser.parseArgument(parameters);
+ if (inputFiles.isEmpty()) {
+ throw new CmdLineException(parser,
+ "asciidoctor: FAILED: input file missing");
+ }
+ } catch (CmdLineException e) {
+ System.err.println(e.getMessage());
+ parser.printUsage(System.err);
+ System.exit(1);
+ return;
+ }
+
+ ZipOutputStream zip = new ZipOutputStream(new FileOutputStream(zipFile));
+ byte[] buf = new byte[BUFSIZ];
+ for (String inputFile : inputFiles) {
+ File tmp = File.createTempFile("doc", ".html");
+ Options options = createOptions(tmp);
+ renderInput(options, inputFile);
+
+ FileInputStream input = new FileInputStream(tmp);
+ int len;
+ zip.putNextEntry(new ZipEntry(mapInFileToOutFile(inputFile)));
+ while ((len = input.read(buf)) > 0) {
+ zip.write(buf, 0, len);
+ }
+ input.close();
+ tmp.delete();
+ zip.closeEntry();
+ }
+ zip.close();
+ }
+
+ private void renderInput(Options options, String inputFile) {
+ Asciidoctor asciidoctor = JRubyAsciidoctor.create();
+ asciidoctor.renderFile(new File(inputFile), options);
+ }
+
+ public static void main(String[] args) {
+ try {
+ new Main().invoke(args);
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+ }
+}
