bazel: bower support
* npm_binary (download tarball packaged npm apps)
* bower_archive (download a zip file, to be put in WORKSPACE)
* bower_component (defining a bower library, with dependency )
* bower_component_bundle (zipping up libraries together)
* js_component (insert plain js file into bower component bundle)
* bower2bazel.py: run bower to find dependencies, generate a .bzl to
define archives and define components
Tested:
python tools/js/bower2bazel.py -w lib/js/bower_archives.bzl -b \
lib/js/bower_components.bzl
bazel build polygerrit-ui:components
unzip -v bazel-bin/polygerrit-ui/components.zip > /tmp/baz
buck build polygerrit-ui:polygerrit_components
unzip -v buck-out/gen/polygerrit-ui/polygerrit_components/polygerrit_components.bower_components.zip > /tmp/buck
diff /tmp/buck /tmp/baz
The diff corresponds to newer file versions pinned through bower2bazel.
Change-Id: I4f33914d4853bcf8afe78b4719d0e0e83b139031
diff --git a/tools/js/BUCK b/tools/js/BUCK
index ba4f19c..9eb0c91 100644
--- a/tools/js/BUCK
+++ b/tools/js/BUCK
@@ -1,14 +1,26 @@
python_binary(
name = 'bower2buck',
main = 'bower2buck.py',
- deps = ['//tools:util'],
+ deps = [
+ '//tools:util',
+ ":bowerutil",
+ ],
visibility = ['PUBLIC'],
)
+python_library(
+ name = 'bowerutil',
+ srcs = [ 'bowerutil.py' ],
+ visibility = [ 'PUBLIC' ],
+)
+
python_binary(
name = 'download_bower',
main = 'download_bower.py',
- deps = ['//tools:util'],
+ deps = [
+ '//tools:util',
+ ":bowerutil",
+ ],
visibility = ['PUBLIC'],
)
diff --git a/tools/js/BUILD b/tools/js/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tools/js/BUILD
diff --git a/tools/js/bower2bazel.py b/tools/js/bower2bazel.py
new file mode 100755
index 0000000..8682012
--- /dev/null
+++ b/tools/js/bower2bazel.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Suggested call sequence:
+
+python tools/js/bower2bazel.py -w lib/js/bower_archives.bzl -b lib/js/bower_components.bzl
+"""
+
+from __future__ import print_function
+
+import atexit
+import collections
+import json
+import hashlib
+import optparse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import glob
+import bowerutil
+
+# Map license names to our canonical names.
+license_map = {
+ "http://polymer.github.io/LICENSE.txt": "polymer",
+ "Apache-2.0": "Apache2.0",
+
+ # TODO(hanwen): remove these, and add appropriate license files under //lib
+ "BSD": "polymer",
+ "MIT": "polymer",
+}
+
+# list of licenses for packages that don't specify one in their bower.json file.
+package_licenses = {
+ "es6-promise": "es6-promise",
+ "fetch": "fetch",
+ "moment": "moment",
+ "page": "page.js",
+ "promise-polyfill": "promise-polyfill",
+ "webcomponentsjs": "polymer", # self-identifies as BSD.
+}
+
+
+def build_bower_json(version_targets):
+ """Generate bower JSON file, return its path."""
+ bower_json = collections.OrderedDict()
+ bower_json['name'] = 'bower2buck-output'
+ bower_json['version'] = '0.0.0'
+ bower_json['description'] = 'Auto-generated bower.json for dependency management'
+ bower_json['private'] = True
+ bower_json['dependencies'] = {}
+
+ for v in version_targets:
+ fn = os.path.join("bazel-out/local-fastbuild/bin", v.lstrip("/").replace(":", "/"))
+ with open(fn) as f:
+ j = json.load(f)
+ if "" in j:
+ # drop dummy entries.
+ del j[""]
+ bower_json['dependencies'].update(j)
+
+ tmpdir = tempfile.mkdtemp()
+ ret = os.path.join(tmpdir, 'bower.json')
+ with open(ret, 'w') as f:
+ json.dump(bower_json, f, indent=2)
+ return ret
+
+
+def bower_command(args):
+ base = subprocess.check_output(["bazel", "info", "output_base"]).strip()
+ exp = os.path.join(base, "external", "bower", "*npm_binary.tgz")
+ fs = sorted(glob.glob(exp))
+ assert len(fs) == 1, "bower tarball not found or have multiple versions %s" % fs
+ return ["python", os.getcwd() + "/tools/js/run_npm_binary.py", sorted(fs)[0]] + args
+
+
+def main(args):
+ opts = optparse.OptionParser()
+ opts.add_option('-w', help='.bzl output for WORKSPACE')
+ opts.add_option('-b', help='.bzl output for //lib:BUILD')
+ opts, args = opts.parse_args()
+
+ target_str = subprocess.check_output([
+ "bazel", "query", "kind(bower_component_bundle, //polygerrit-ui/...)"])
+ seed_str = subprocess.check_output([
+ "bazel", "query", "attr(seed, 1, kind(bower_component, deps(//polygerrit-ui/...)))"])
+ targets = [s for s in target_str.split('\n') if s]
+ seeds = [s for s in seed_str.split('\n') if s]
+ prefix = "//lib/js:"
+ non_seeds = [s for s in seeds if not s.startswith(prefix)]
+ assert not non_seeds, non_seeds
+ seeds = set([s[len(prefix):] for s in seeds])
+
+ version_targets = [t + "-versions.json" for t in targets]
+
+ subprocess.check_call(['bazel', 'build'] + version_targets)
+ bower_json_path = build_bower_json(version_targets)
+ dir = os.path.dirname(bower_json_path)
+ cmd = bower_command(["install"])
+
+ build_out = sys.stdout
+ if opts.b:
+ build_out = open(opts.b + ".tmp", 'w')
+
+ ws_out = sys.stdout
+ if opts.b:
+ ws_out = open(opts.w + ".tmp", 'w')
+
+ header = """# DO NOT EDIT
+# generated with the following command:
+#
+# %s
+#
+
+""" % ' '.join(sys.argv)
+
+ ws_out.write(header)
+ build_out.write(header)
+
+ oldwd = os.getcwd()
+ os.chdir(dir)
+ subprocess.check_call(cmd)
+
+ interpret_bower_json(seeds, ws_out, build_out)
+ ws_out.close()
+ build_out.close()
+
+ os.chdir(oldwd)
+ os.rename(opts.w + ".tmp", opts.w)
+ os.rename(opts.b + ".tmp", opts.b)
+
+
+def dump_workspace(data, seeds, out):
+ out.write('load("//tools/bzl:js.bzl", "bower_archive")\n')
+ out.write('def load_bower_archives():\n')
+
+ for d in data:
+ if d["name"] in seeds:
+ continue
+ out.write(""" bower_archive(
+ name = "%(name)s",
+ package = "%(normalized-name)s",
+ version = "%(version)s",
+ sha1 = "%(bazel-sha1)s")
+""" % d)
+
+
+def dump_build(data, seeds, out):
+ out.write('load("//tools/bzl:js.bzl", "bower_component")\n')
+ out.write('def define_bower_components():\n')
+ for d in data:
+ out.write(" bower_component(\n")
+ out.write(" name = \"%s\",\n" % d["name"])
+ out.write(" license = \"//lib:LICENSE-%s\",\n" % d["bazel-license"])
+ deps = sorted(d.get("dependencies", {}).keys())
+ if deps:
+ if len(deps) == 1:
+ out.write(" deps = [ \":%s\" ],\n" % deps[0])
+ else:
+ out.write(" deps = [\n")
+ for dep in deps:
+ out.write(" \":%s\",\n" % dep)
+ out.write(" ],\n")
+ if d["name"] in seeds:
+ out.write(" seed = True,\n")
+ out.write(" )\n")
+ # done
+
+
+def interpret_bower_json(seeds, ws_out, build_out):
+ out = subprocess.check_output(["find", "bower_components/", "-name", ".bower.json"])
+
+ data = []
+ for f in sorted(out.split('\n')):
+ if not f:
+ continue
+ pkg = json.load(open(f))
+ pkg_name = pkg["name"]
+
+ pkg["bazel-sha1"] = bowerutil.hash_bower_component(
+ hashlib.sha1(), os.path.dirname(f)).hexdigest()
+ license = pkg.get("license", None)
+ if type(license) == type([]):
+ # WTF? Some package specify a list of licenses. ("GPL", "MIT")
+ pick = license[0]
+ sys.stderr.write("package %s has multiple licenses: %s, picking %s" % (pkg_name, ", ".join(license), pick))
+ license = pick
+
+ if license:
+ license = license_map.get(license, license)
+ else:
+ if pkg_name not in package_licenses:
+ msg = "package %s does not specify license." % pkg_name
+ sys.stderr.write(msg)
+ raise Exception(msg)
+ license = package_licenses[pkg_name]
+
+ pkg["bazel-license"] = license
+
+ # TODO(hanwen): bower packages can also have 'fully qualified'
+ # names, ("PolymerElements/iron-ajax") as well as short names
+ # ("iron-ajax"). It is possible for bower.json files to refer to
+ # long names as their dependencies. If any package does this, we
+ # will have to either 1) strip off the prefix (typically github
+ # user?), or 2) build a map of short name <=> fully qualified
+ # name. For now, we just ignore the problem.
+ pkg["normalized-name"] = pkg["name"]
+ data.append(pkg)
+
+ dump_workspace(data, seeds, ws_out)
+ dump_build(data, seeds, build_out)
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
diff --git a/tools/js/bower2buck.py b/tools/js/bower2buck.py
index 81072da..57da475 100755
--- a/tools/js/bower2buck.py
+++ b/tools/js/bower2buck.py
@@ -78,7 +78,7 @@
self.version = bower_json['version']
self.deps = bower_json.get('dependencies', {})
self.license = bower_json.get('license', 'NO LICENSE')
- self.sha1 = util.hash_bower_component(
+ self.sha1 = bowerutil.hash_bower_component(
hashlib.sha1(), os.path.dirname(bower_json_path)).hexdigest()
def to_rule(self, packages):
@@ -106,6 +106,7 @@
def build_bower_json(targets, buck_out):
+ """create bower.json so 'bower install' fetches transitive deps"""
bower_json = collections.OrderedDict()
bower_json['name'] = 'bower2buck-output'
bower_json['version'] = '0.0.0'
@@ -117,6 +118,9 @@
['buck', 'query', '-v', '0',
"filter('__download_bower', deps(%s))" % '+'.join(targets)],
env=BUCK_ENV)
+
+ # __bower_version contains the version number coming from version
+ # attr in BUCK/BUILD
deps = deps.replace('__download_bower', '__bower_version').split()
subprocess.check_call(['buck', 'build'] + deps, env=BUCK_ENV)
diff --git a/tools/js/bowerutil.py b/tools/js/bowerutil.py
new file mode 100644
index 0000000..eb8893b
--- /dev/null
+++ b/tools/js/bowerutil.py
@@ -0,0 +1,47 @@
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from os import path
+
+
+def hash_bower_component(hash_obj, path):
+ """Hash the contents of a bower component directory.
+
+ This is a stable hash of a directory downloaded with `bower install`, minus
+ the .bower.json file, which is autogenerated each time by bower. Used in lieu
+ of hashing a zipfile of the contents, since zipfiles are difficult to hash in
+ a stable manner.
+
+ Args:
+ hash_obj: an open hash object, e.g. hashlib.sha1().
+ path: path to the directory to hash.
+
+ Returns:
+ The passed-in hash_obj.
+ """
+ if not os.path.isdir(path):
+ raise ValueError('Not a directory: %s' % path)
+
+ path = os.path.abspath(path)
+ for root, dirs, files in os.walk(path):
+ dirs.sort()
+ for f in sorted(files):
+ if f == '.bower.json':
+ continue
+ p = os.path.join(root, f)
+ hash_obj.update(p[len(path)+1:])
+ hash_obj.update(open(p).read())
+
+ return hash_obj
diff --git a/tools/js/download_bower.py b/tools/js/download_bower.py
old mode 100644
new mode 100755
index bcc417c..f5b7bf5
--- a/tools/js/download_bower.py
+++ b/tools/js/download_bower.py
@@ -23,8 +23,7 @@
import subprocess
import sys
-from tools import util
-
+import bowerutil
CACHE_DIR = os.path.expanduser(os.path.join(
'~', '.gerritcodereview', 'buck-cache', 'downloaded-artifacts'))
@@ -39,16 +38,20 @@
def bower_info(bower, name, package, version):
cmd = bower_cmd(bower, '-l=error', '-j',
'info', '%s#%s' % (package, version))
- p = subprocess.Popen(cmd , stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ try:
+ p = subprocess.Popen(cmd , stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ except:
+ sys.stderr.write("error executing: %s\n" % ' '.join(cmd))
+ raise
out, err = p.communicate()
if p.returncode:
sys.stderr.write(err)
- raise OSError('Command failed: %s' % cmd)
+ raise OSError('Command failed: %s' % ' '.join(cmd))
try:
info = json.loads(out)
except ValueError:
- raise ValueError('invalid JSON from %s:\n%s' % (cmd, out))
+ raise ValueError('invalid JSON from %s:\n%s' % (" ".join(cmd), out))
info_name = info.get('name')
if info_name != name:
raise ValueError('expected package name %s, got: %s' % (name, info_name))
@@ -82,7 +85,11 @@
opts.add_option('-v', help='version number')
opts.add_option('-s', help='expected content sha1')
opts.add_option('-o', help='output file location')
- opts, _ = opts.parse_args()
+ opts, args_ = opts.parse_args(args)
+
+ assert opts.p
+ assert opts.v
+ assert opts.n
cwd = os.getcwd()
outzip = os.path.join(cwd, opts.o)
@@ -100,7 +107,7 @@
if opts.s:
path = os.path.join(bc, opts.n)
- sha1 = util.hash_bower_component(hashlib.sha1(), path).hexdigest()
+ sha1 = bowerutil.hash_bower_component(hashlib.sha1(), path).hexdigest()
if opts.s != sha1:
print((
'%s#%s:\n'
diff --git a/tools/js/run_npm_binary.py b/tools/js/run_npm_binary.py
index d76eff5..d769b98 100644
--- a/tools/js/run_npm_binary.py
+++ b/tools/js/run_npm_binary.py
@@ -25,8 +25,6 @@
import tarfile
import tempfile
-from tools import util
-
def extract(path, outdir, bin):
if os.path.exists(os.path.join(outdir, bin)):
@@ -59,19 +57,21 @@
# finished.
extract_one(tar.getmember(bin))
-
def main(args):
path = args[0]
suffix = '.npm_binary.tgz'
tgz = os.path.basename(path)
+
parts = tgz[:-len(suffix)].split('@')
if not tgz.endswith(suffix) or len(parts) != 2:
print('usage: %s <path/to/npm_binary>' % sys.argv[0], file=sys.stderr)
return 1
- name, version = parts
- sha1 = util.hash_file(hashlib.sha1(), path).hexdigest()
+ name, _ = parts
+
+ # Avoid importing from gerrit because we don't want to depend on the right CWD.
+ sha1 = hashlib.sha1(open(path, 'rb').read()).hexdigest()
outdir = '%s-%s' % (path[:-len(suffix)], sha1)
rel_bin = os.path.join('package', 'bin', name)
bin = os.path.join(outdir, rel_bin)
