Add web worker infrastructure and a hljs worker
We are adding hljs-worker.ts, which is capable of highlighting files
in separate OS threads, thus improving the diff rendering performance.
Workers are separate bundles, so we are adapting the polygerrit rule
to also create a rollup bundle for workers and place them in the
resulting zip file for deployment.
We have decided to keep the worker source code withing the app/ tree,
because that makes it easier to test and re-use some common types and
utilities. As long as workers do not pull in large deps or DOM related
deps that is not problematic. And obviously the main app should not
depend on any code in the workers/ subdirectory.
Change-Id: Ie47b72b08079d55b5f885a9141e0804c128af446
diff --git a/polygerrit-ui/app/BUILD b/polygerrit-ui/app/BUILD
index d66c18e..8e32258 100644
--- a/polygerrit-ui/app/BUILD
+++ b/polygerrit-ui/app/BUILD
@@ -22,6 +22,7 @@
"styles",
"types",
"utils",
+ "workers",
]
ts_config(
diff --git a/polygerrit-ui/app/rules.bzl b/polygerrit-ui/app/rules.bzl
index 401c0c3..01bb31f 100644
--- a/polygerrit-ui/app/rules.bzl
+++ b/polygerrit-ui/app/rules.bzl
@@ -32,6 +32,18 @@
],
)
+ rollup_bundle(
+ name = "syntax-worker",
+ srcs = [app_name + "-full-src"],
+ config_file = ":rollup.config.js",
+ entry_point = "_pg_ts_out/workers/syntax-worker.js",
+ rollup_bin = "//tools/node_tools:rollup-bin",
+ sourcemap = "hidden",
+ deps = [
+ "@tools_npm//rollup-plugin-node-resolve",
+ ],
+ )
+
native.filegroup(
name = name + "_app_sources",
srcs = [
@@ -45,6 +57,13 @@
)
native.filegroup(
+ name = name + "_worker_sources",
+ srcs = [
+ "syntax-worker.js",
+ ],
+ )
+
+ native.filegroup(
name = name + "_top_sources",
srcs = [
"favicon.ico",
@@ -59,6 +78,7 @@
name + "_app_sources",
name + "_css_sources",
name + "_top_sources",
+ name + "_worker_sources",
"//lib/fonts:robotofonts",
"//lib/js:highlightjs__files",
"@ui_npm//:node_modules/@webcomponents/webcomponentsjs/webcomponents-lite.js",
@@ -70,11 +90,12 @@
outs = outs,
cmd = " && ".join([
"FONT_DIR=$$(dirname $(location @ui_npm//:node_modules/@polymer/font-roboto-local/package.json))/fonts",
- "mkdir -p $$TMP/polygerrit_ui/{styles/themes,fonts/{roboto,robotomono},bower_components/{highlightjs,webcomponentsjs,resemblejs},elements}",
+ "mkdir -p $$TMP/polygerrit_ui/{workers,styles/themes,fonts/{roboto,robotomono},bower_components/{highlightjs,webcomponentsjs,resemblejs},elements}",
"for f in $(locations " + name + "_app_sources); do ext=$${f##*.}; cp -p $$f $$TMP/polygerrit_ui/elements/" + app_name + ".$$ext; done",
"cp $(locations //lib/fonts:robotofonts) $$TMP/polygerrit_ui/fonts/",
"for f in $(locations " + name + "_top_sources); do cp $$f $$TMP/polygerrit_ui/; done",
"for f in $(locations " + name + "_css_sources); do cp $$f $$TMP/polygerrit_ui/styles; done",
+ "for f in $(locations " + name + "_worker_sources); do cp $$f $$TMP/polygerrit_ui/workers; done",
"for f in $(locations //lib/js:highlightjs__files); do cp $$f $$TMP/polygerrit_ui/bower_components/highlightjs/ ; done",
"cp $(location @ui_npm//:node_modules/@webcomponents/webcomponentsjs/webcomponents-lite.js) $$TMP/polygerrit_ui/bower_components/webcomponentsjs/webcomponents-lite.js",
"cp $(location @ui_npm//:node_modules/@webcomponents/webcomponentsjs/webcomponents-lite.js.map) $$TMP/polygerrit_ui/bower_components/webcomponentsjs/webcomponents-lite.js.map",
diff --git a/polygerrit-ui/app/tsconfig.json b/polygerrit-ui/app/tsconfig.json
index a335926..ac26c6d 100644
--- a/polygerrit-ui/app/tsconfig.json
+++ b/polygerrit-ui/app/tsconfig.json
@@ -95,6 +95,7 @@
"types/**/*",
"utils/**/*",
"test/**/*",
+ "workers/**/*",
"tmpl_out/**/*" //Created by template checker in dev-mode
]
}
diff --git a/polygerrit-ui/app/tsconfig_bazel.json b/polygerrit-ui/app/tsconfig_bazel.json
index cd83fc0..dba3060 100644
--- a/polygerrit-ui/app/tsconfig_bazel.json
+++ b/polygerrit-ui/app/tsconfig_bazel.json
@@ -22,10 +22,11 @@
"services/**/*",
"styles/**/*",
"types/**/*",
- "utils/**/*"
+ "utils/**/*",
+ "workers/**/*"
],
"exclude": [
"**/*_test.ts",
"**/*_test.js"
]
-}
+}
\ No newline at end of file
diff --git a/polygerrit-ui/app/tsconfig_bazel_test.json b/polygerrit-ui/app/tsconfig_bazel_test.json
index be3c934..78bb350 100644
--- a/polygerrit-ui/app/tsconfig_bazel_test.json
+++ b/polygerrit-ui/app/tsconfig_bazel_test.json
@@ -7,7 +7,9 @@
"../../external/ui_dev_npm/node_modules/@types"
],
"paths": {
- "@polymer/iron-test-helpers/*": ["../../ui_dev_npm/node_modules/@polymer/iron-test-helpers/*"]
+ "@polymer/iron-test-helpers/*": [
+ "../../ui_dev_npm/node_modules/@polymer/iron-test-helpers/*"
+ ]
}
},
"include": [
@@ -25,9 +27,10 @@
"scripts/**/*",
"services/**/*",
"styles/**/*",
+ "test/**/*",
"types/**/*",
"utils/**/*",
- "test/**/*"
+ "workers/**/*"
],
"exclude": []
-}
+}
\ No newline at end of file
diff --git a/polygerrit-ui/app/types/syntax-worker-api.ts b/polygerrit-ui/app/types/syntax-worker-api.ts
new file mode 100644
index 0000000..c9c2540
--- /dev/null
+++ b/polygerrit-ui/app/types/syntax-worker-api.ts
@@ -0,0 +1,90 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * This file defines the API of syntax-worker, which is a web worker for syntax
+ * highlighting based on the HighlightJS library.
+ *
+ * Workers communicate via `postMessage(e)` and `onMessage(e)` where `e` is a
+ * MessageEvent.
+ *
+ * SyntaxWorker expects incoming messages to be of type
+ * `MessageEvent<SyntaxWorkerMessage>`. And outgoing messages will be of type
+ * `MessageEvent<SyntaxWorkerResult>`.
+ */
+
+/** Type of incoming messages for SyntaxWorker. */
+export enum SyntaxWorkerMessageType {
+ INIT,
+ REQUEST,
+}
+
+/** Incoming message for SyntaxWorker. */
+export interface SyntaxWorkerMessage {
+ type: SyntaxWorkerMessageType;
+}
+
+/**
+ * Requests the worker to import the HighlightJS lib from the given URL and
+ * initializes and configures it. Has to be called once before you can send
+ * a SyntaxWorkerRequest message.
+ */
+export interface SyntaxWorkerInit extends SyntaxWorkerMessage {
+ type: SyntaxWorkerMessageType.INIT;
+ url: string;
+}
+
+export function isInit(
+ x: SyntaxWorkerMessage | undefined
+): x is SyntaxWorkerInit {
+ return !!x && x.type === SyntaxWorkerMessageType.INIT;
+}
+
+/**
+ * Requests the worker to highlight the given code. The worker must have been
+ * initialized before.
+ */
+export interface SyntaxWorkerRequest extends SyntaxWorkerMessage {
+ type: SyntaxWorkerMessageType.REQUEST;
+ language: string;
+ code: string;
+}
+
+export function isRequest(
+ x: SyntaxWorkerMessage | undefined
+): x is SyntaxWorkerRequest {
+ return !!x && x.type === SyntaxWorkerMessageType.REQUEST;
+}
+
+/** Type of outgoing messages of SyntaxWorker. */
+export interface SyntaxWorkerResult {
+ /** Unset or undefined means "success". */
+ error?: string;
+ /**
+ * Returned by SyntaxWorkerRequest calls. Every line gets its own array of
+ * ranges. `ranges[0]` are the ranges for line 1. Every line has an array,
+ * which may be empty. All ranges are guaranteed to be closed.
+ */
+ ranges?: SyntaxLayerLine[];
+}
+
+/** Ranges for one line. */
+export interface SyntaxLayerLine {
+ ranges: SyntaxLayerRange[];
+}
+
+/** Can be used for `length` in SyntaxLayerRange. */
+export const UNCLOSED = -1;
+
+/** Range of characters in a line to be syntax highlighted. */
+export interface SyntaxLayerRange {
+ /** 1-based inclusive. */
+ start: number;
+ /** Can only be UNCLOSED during processing. */
+ length: number;
+ /** HighlightJS specific names, e.g. 'literal'. */
+ className: string;
+}
diff --git a/polygerrit-ui/app/types/types.ts b/polygerrit-ui/app/types/types.ts
index c6137eb..ab78015 100644
--- a/polygerrit-ui/app/types/types.ts
+++ b/polygerrit-ui/app/types/types.ts
@@ -166,7 +166,7 @@
languageName: string,
code: string,
ignore_illegals: boolean,
- continuation: unknown
+ continuation?: unknown
): HighlightJSResult;
}
diff --git a/polygerrit-ui/app/utils/hljs-util.ts b/polygerrit-ui/app/utils/syntax-util.ts
similarity index 88%
rename from polygerrit-ui/app/utils/hljs-util.ts
rename to polygerrit-ui/app/utils/syntax-util.ts
index 1bd2072..318d46a 100644
--- a/polygerrit-ui/app/utils/hljs-util.ts
+++ b/polygerrit-ui/app/utils/syntax-util.ts
@@ -3,11 +3,16 @@
* Copyright 2022 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
+import {
+ SyntaxLayerLine,
+ SyntaxLayerRange,
+ UNCLOSED,
+} from '../types/syntax-worker-api';
/**
* Utilities related to working with the HighlightJS syntax highlighting lib.
*
- * Note that this utility is mostly used by the hljs-worker, which is a Web
+ * Note that this utility is mostly used by the syntax-worker, which is a Web
* Worker and can thus not depend on document, the DOM or any related
* functionality.
*/
@@ -20,19 +25,6 @@
const openingSpan = new RegExp('<span class="(.*?)">');
const closingSpan = new RegExp('</span>');
-/** Can be used for `length` in SyntaxLayerRange. */
-const UNCLOSED = -1;
-
-/** Range of characters in a line to be syntax highlighted. */
-export interface SyntaxLayerRange {
- /** 1-based inclusive. */
- start: number;
- /** Can only be UNCLOSED during processing. */
- length: number;
- /** HighlightJS specific names, e.g. 'literal'. */
- className: string;
-}
-
/**
* HighlightJS produces one long HTML string with HTML elements spanning
* multiple lines. <gr-diff> is line based, needs all elements closed at the end
@@ -44,16 +36,16 @@
*/
export function highlightedStringToRanges(
highlightedCode: string
-): SyntaxLayerRange[][] {
+): SyntaxLayerLine[] {
// What the function eventually returns.
- const rangesPerLine: SyntaxLayerRange[][] = [];
+ const rangesPerLine: SyntaxLayerLine[] = [];
// The unclosed ranges that are carried over from one line to the next.
let carryOverRanges: SyntaxLayerRange[] = [];
for (let line of highlightedCode.split('\n')) {
const ranges: SyntaxLayerRange[] = [...carryOverRanges];
carryOverRanges = [];
- rangesPerLine.push(ranges);
+ rangesPerLine.push({ranges});
// Remove all span tags one after another from left to right.
// For each opening <span ...> push a new (unclosed) range.
diff --git a/polygerrit-ui/app/utils/hljs-util_test.ts b/polygerrit-ui/app/utils/syntax-util_test.ts
similarity index 62%
rename from polygerrit-ui/app/utils/hljs-util_test.ts
rename to polygerrit-ui/app/utils/syntax-util_test.ts
index 3c577ca..fef908a 100644
--- a/polygerrit-ui/app/utils/hljs-util_test.ts
+++ b/polygerrit-ui/app/utils/syntax-util_test.ts
@@ -4,14 +4,14 @@
* SPDX-License-Identifier: Apache-2.0
*/
import '../test/common-test-setup-karma';
-import './hljs-util';
+import './syntax-util';
import {
highlightedStringToRanges,
removeFirstSpan,
SpanType,
-} from './hljs-util';
+} from './syntax-util';
-suite('file hljs-util', () => {
+suite('file syntax-util', () => {
suite('function removeFirstSpan()', () => {
test('no matches', async () => {
assert.isUndefined(removeFirstSpan(''));
@@ -44,23 +44,26 @@
suite('function highlightedStringToRanges()', () => {
test('no ranges', async () => {
- assert.deepEqual(highlightedStringToRanges(''), [[]]);
- assert.deepEqual(highlightedStringToRanges('\n'), [[], []]);
+ assert.deepEqual(highlightedStringToRanges(''), [{ranges: []}]);
+ assert.deepEqual(highlightedStringToRanges('\n'), [
+ {ranges: []},
+ {ranges: []},
+ ]);
assert.deepEqual(highlightedStringToRanges('asdf\nasdf\nasdf'), [
- [],
- [],
- [],
+ {ranges: []},
+ {ranges: []},
+ {ranges: []},
]);
});
test('one line, one span', async () => {
assert.deepEqual(
highlightedStringToRanges('asdf<span class="c">qwer</span>asdf'),
- [[{start: 4, length: 4, className: 'c'}]]
+ [{ranges: [{start: 4, length: 4, className: 'c'}]}]
);
assert.deepEqual(
highlightedStringToRanges('<span class="d">asdfqwer</span>'),
- [[{start: 0, length: 8, className: 'd'}]]
+ [{ranges: [{start: 0, length: 8, className: 'd'}]}]
);
});
@@ -70,10 +73,12 @@
'asdf<span class="c">qwer</span>zxcv<span class="d">qwer</span>asdf'
),
[
- [
- {start: 4, length: 4, className: 'c'},
- {start: 12, length: 4, className: 'd'},
- ],
+ {
+ ranges: [
+ {start: 4, length: 4, className: 'c'},
+ {start: 12, length: 4, className: 'd'},
+ ],
+ },
]
);
});
@@ -84,10 +89,12 @@
'asdf<span class="c">qwer<span class="d">zxcv</span>qwer</span>asdf'
),
[
- [
- {start: 4, length: 12, className: 'c'},
- {start: 8, length: 4, className: 'd'},
- ],
+ {
+ ranges: [
+ {start: 4, length: 12, className: 'c'},
+ {start: 8, length: 4, className: 'd'},
+ ],
+ },
]
);
});
@@ -99,8 +106,8 @@
'asd<span class="d">qwe</span>asd'
),
[
- [{start: 4, length: 4, className: 'c'}],
- [{start: 3, length: 3, className: 'd'}],
+ {ranges: [{start: 4, length: 4, className: 'c'}]},
+ {ranges: [{start: 3, length: 3, className: 'd'}]},
]
);
});
@@ -111,8 +118,8 @@
'asdf<span class="c">qwer\n' + 'asdf</span>qwer'
),
[
- [{start: 4, length: 4, className: 'c'}],
- [{start: 0, length: 4, className: 'c'}],
+ {ranges: [{start: 4, length: 4, className: 'c'}]},
+ {ranges: [{start: 0, length: 4, className: 'c'}]},
]
);
});
@@ -124,14 +131,18 @@
'asdf</span>qwer</span>zxcv'
),
[
- [
- {start: 4, length: 8, className: 'c'},
- {start: 8, length: 4, className: 'd'},
- ],
- [
- {start: 0, length: 8, className: 'c'},
- {start: 0, length: 4, className: 'd'},
- ],
+ {
+ ranges: [
+ {start: 4, length: 8, className: 'c'},
+ {start: 8, length: 4, className: 'd'},
+ ],
+ },
+ {
+ ranges: [
+ {start: 0, length: 8, className: 'c'},
+ {start: 0, length: 4, className: 'd'},
+ ],
+ },
]
);
});
@@ -145,16 +156,20 @@
'asdf</span>qwer'
),
[
- [{start: 4, length: 4, className: 'c'}],
- [
- {start: 0, length: 8, className: 'c'},
- {start: 4, length: 4, className: 'd'},
- ],
- [
- {start: 0, length: 8, className: 'c'},
- {start: 0, length: 4, className: 'd'},
- ],
- [{start: 0, length: 4, className: 'c'}],
+ {ranges: [{start: 4, length: 4, className: 'c'}]},
+ {
+ ranges: [
+ {start: 0, length: 8, className: 'c'},
+ {start: 4, length: 4, className: 'd'},
+ ],
+ },
+ {
+ ranges: [
+ {start: 0, length: 8, className: 'c'},
+ {start: 0, length: 4, className: 'd'},
+ ],
+ },
+ {ranges: [{start: 0, length: 4, className: 'c'}]},
]
);
});
diff --git a/polygerrit-ui/app/workers/syntax-worker.ts b/polygerrit-ui/app/workers/syntax-worker.ts
new file mode 100644
index 0000000..85a0fb4
--- /dev/null
+++ b/polygerrit-ui/app/workers/syntax-worker.ts
@@ -0,0 +1,84 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import {HighlightJS} from '../types/types';
+import {
+ SyntaxWorkerMessage,
+ SyntaxWorkerResult,
+ isRequest,
+ isInit,
+} from '../types/syntax-worker-api';
+import {highlightedStringToRanges} from '../utils/syntax-util';
+
+// This is an entry point file of a bundle. Keep free of exports!
+
+/**
+ * This is a web worker for calling the HighlightJS library for syntax
+ * highlighting. Files can be large and highlighting does not require
+ * the `document` or the `DOM`, so it is a perfect fit for a web worker.
+ *
+ * This file is a just a hub hooking into the web worker API. The message
+ * events for communicating with the main app are defined in the file
+ * `types/worker-api.ts`. And the `meat` of the computation is done in the
+ * file `syntax-util.ts`.
+ */
+
+/**
+ * `self` is for a worker what `window` is for the web app. It is called
+ * the `DedicatedWorkerGlobalScope`, see
+ * https://developer.mozilla.org/en-US/docs/Web/API/DedicatedWorkerGlobalScope
+ *
+ * Once imported the HighlightJS lib exposes its functionality via the global
+ * `hljs` variable.
+ */
+type Context = Worker & {hljs?: HighlightJS};
+const ctx: Context = self as unknown as Context;
+
+/**
+ * We are encapsulating the web worker API here, so this is the only place
+ * where you need to know about it and the MessageEvents in this file.
+ */
+ctx.onmessage = function (e: MessageEvent<SyntaxWorkerMessage>) {
+ try {
+ const message = e.data;
+ if (isInit(message)) {
+ worker.init(message.url);
+ const result: SyntaxWorkerResult = {ranges: []};
+ ctx.postMessage(result);
+ }
+ if (isRequest(message)) {
+ const ranges = worker.highlight(message.language, message.code);
+ const result: SyntaxWorkerResult = {ranges};
+ ctx.postMessage(result);
+ }
+ } catch (err) {
+ let error = 'syntax worker error';
+ if (err instanceof Error) error = err.message;
+ const result: SyntaxWorkerResult = {error, ranges: []};
+ ctx.postMessage(result);
+ }
+};
+
+class SyntaxWorker {
+ private highlightJsLib?: HighlightJS;
+
+ init(highlightJsLibUrl: string) {
+ importScripts(highlightJsLibUrl);
+ if (!ctx.hljs) {
+ throw new Error('HighlightJS lib not available after import');
+ }
+ this.highlightJsLib = ctx.hljs;
+ this.highlightJsLib.configure({classPrefix: ''});
+ }
+
+ highlight(language: string, code: string) {
+ if (!this.highlightJsLib) throw new Error('worker not initialized');
+ const highlight = this.highlightJsLib.highlight(language, code, true);
+ return highlightedStringToRanges(highlight.value);
+ }
+}
+
+/** Singleton instance being referenced in `onmessage` function above. */
+const worker = new SyntaxWorker();
