| from __future__ import with_statement |
| |
| import __future__ |
| import functools |
| import imp |
| import inspect |
| import json |
| from pathlib import Path |
| import optparse |
| import os |
| import os.path |
| import sys |
| |
| |
| # When build files are executed, the functions in this file tagged with |
| # @provide_for_build will be provided in the build file's local symbol table. |
| # |
| # When these functions are called from a build file, they will be passed |
| # a keyword parameter, build_env, which is a object with information about |
| # the environment of the build file which is currently being processed. |
| # It contains the following attributes: |
| # |
| # "dirname" - The directory containing the build file. |
| # |
| # "base_path" - The base path of the build file. |
| |
| BUILD_FUNCTIONS = [] |
| |
| |
| class BuildContextType(object): |
| """ |
| Identifies the type of input file to the processor. |
| """ |
| |
| BUILD_FILE = 'build_file' |
| INCLUDE = 'include' |
| |
| |
| class BuildFileContext(object): |
| """ |
| The build context used when processing a build file. |
| """ |
| |
| type = BuildContextType.BUILD_FILE |
| |
| def __init__(self, base_path, dirname, allow_empty_globs): |
| self.globals = {} |
| self.includes = set() |
| self.base_path = base_path |
| self.dirname = dirname |
| self.allow_empty_globs = allow_empty_globs |
| self.rules = {} |
| |
| |
| class IncludeContext(object): |
| """ |
| The build context used when processing an include. |
| """ |
| |
| type = BuildContextType.INCLUDE |
| |
| def __init__(self): |
| self.globals = {} |
| self.includes = set() |
| |
| |
| class LazyBuildEnvPartial(object): |
| """Pairs a function with a build environment in which it will be executed. |
| |
| Note that while the function is specified via the constructor, the build |
| environment must be assigned after construction, for the build environment |
| currently being used. |
| |
| To call the function with its build environment, use the invoke() method of |
| this class, which will forward the arguments from invoke() to the |
| underlying function. |
| """ |
| |
| def __init__(self, func): |
| self.func = func |
| self.build_env = None |
| |
| def invoke(self, *args, **kwargs): |
| """Invokes the bound function injecting 'build_env' into **kwargs.""" |
| updated_kwargs = kwargs.copy() |
| updated_kwargs.update({'build_env': self.build_env}) |
| return self.func(*args, **updated_kwargs) |
| |
| |
| def provide_for_build(func): |
| BUILD_FUNCTIONS.append(func) |
| return func |
| |
| |
| def add_rule(rule, build_env): |
| assert build_env.type == BuildContextType.BUILD_FILE, ( |
| "Cannot use `{}()` at the top-level of an included file." |
| .format(rule['type'])) |
| |
| # Include the base path of the BUILD file so the reader consuming this |
| # JSON will know which BUILD file the rule came from. |
| if 'name' not in rule: |
| raise ValueError( |
| 'rules must contain the field \'name\'. Found %s.' % rule) |
| rule_name = rule['name'] |
| if rule_name in build_env.rules: |
| raise ValueError('Duplicate rule definition found. Found %s and %s' % |
| (rule, build_env.rules[rule_name])) |
| rule['buck.base_path'] = build_env.base_path |
| build_env.rules[rule_name] = rule |
| |
| |
| @provide_for_build |
| def glob(includes, excludes=[], include_dotfiles=False, build_env=None): |
| assert build_env.type == BuildContextType.BUILD_FILE, ( |
| "Cannot use `glob()` at the top-level of an included file.") |
| |
| search_base = Path(build_env.dirname) |
| return glob_internal( |
| includes, |
| excludes, |
| include_dotfiles, |
| build_env.allow_empty_globs, |
| search_base) |
| |
| |
| def glob_internal(includes, excludes, include_dotfiles, allow_empty, search_base): |
| # Ensure the user passes lists of strings rather than just a string. |
| assert not isinstance(includes, basestring), \ |
| "The first argument to glob() must be a list of strings." |
| assert not isinstance(excludes, basestring), \ |
| "The excludes argument must be a list of strings." |
| |
| def includes_iterator(): |
| for pattern in includes: |
| for path in search_base.glob(pattern): |
| # TODO(user): Handle hidden files on Windows. |
| if path.is_file() and (include_dotfiles or not path.name.startswith('.')): |
| yield path.relative_to(search_base) |
| |
| def is_special(pat): |
| return "*" in pat or "?" in pat or "[" in pat |
| |
| non_special_excludes = set() |
| match_excludes = set() |
| for pattern in excludes: |
| if is_special(pattern): |
| match_excludes.add(pattern) |
| else: |
| non_special_excludes.add(pattern) |
| |
| def exclusion(path): |
| if path.as_posix() in non_special_excludes: |
| return True |
| for pattern in match_excludes: |
| result = path.match(pattern, match_entire=True) |
| if result: |
| return True |
| return False |
| |
| results = sorted(set([str(p) for p in includes_iterator() if not exclusion(p)])) |
| assert allow_empty or results, ( |
| "glob(includes={includes}, excludes={excludes}, include_dotfiles={include_dotfiles}) " + |
| "returned no results. (allow_empty_globs is set to false in the Buck " + |
| "configuration)").format( |
| includes=includes, |
| excludes=excludes, |
| include_dotfiles=include_dotfiles) |
| |
| return results |
| |
| |
| @provide_for_build |
| def get_base_path(build_env=None): |
| """Get the base path to the build file that was initially evaluated. |
| |
| This function is intended to be used from within a build defs file that |
| likely contains macros that could be called from any build file. |
| Such macros may need to know the base path of the file in which they |
| are defining new build rules. |
| |
| Returns: a string, such as "java/com/facebook". Note there is no |
| trailing slash. The return value will be "" if called from |
| the build file in the root of the project. |
| """ |
| assert build_env.type == BuildContextType.BUILD_FILE, ( |
| "Cannot use `get_base_path()` at the top-level of an included file.") |
| return build_env.base_path |
| |
| |
| @provide_for_build |
| def add_deps(name, deps=[], build_env=None): |
| assert build_env.type == BuildContextType.BUILD_FILE, ( |
| "Cannot use `add_deps()` at the top-level of an included file.") |
| |
| if name not in build_env.rules: |
| raise ValueError( |
| 'Invoked \'add_deps\' on non-existent rule %s.' % name) |
| |
| rule = build_env.rules[name] |
| if 'deps' not in rule: |
| raise ValueError( |
| 'Invoked \'add_deps\' on rule %s that has no \'deps\' field' |
| % name) |
| rule['deps'] = rule['deps'] + deps |
| |
| |
| class BuildFileProcessor(object): |
| |
| def __init__(self, project_root, build_file_name, allow_empty_globs, implicit_includes=[]): |
| self._cache = {} |
| self._build_env_stack = [] |
| |
| self._project_root = project_root |
| self._build_file_name = build_file_name |
| self._implicit_includes = implicit_includes |
| self._allow_empty_globs = allow_empty_globs |
| |
| lazy_functions = {} |
| for func in BUILD_FUNCTIONS: |
| func_with_env = LazyBuildEnvPartial(func) |
| lazy_functions[func.__name__] = func_with_env |
| self._functions = lazy_functions |
| |
| def _merge_globals(self, src, dst): |
| """ |
| Copy the global definitions from one globals dict to another. |
| |
| Ignores special attributes and attributes starting with '_', which |
| typically denote module-level private attributes. |
| """ |
| |
| hidden = set([ |
| 'include_defs', |
| ]) |
| |
| for key, val in src.iteritems(): |
| if not key.startswith('_') and key not in hidden: |
| dst[key] = val |
| |
| def _update_functions(self, build_env): |
| """ |
| Updates the build functions to use the given build context when called. |
| """ |
| |
| for function in self._functions.itervalues(): |
| function.build_env = build_env |
| |
| def _install_functions(self, namespace): |
| """ |
| Installs the build functions, by their name, into the given namespace. |
| """ |
| |
| for name, function in self._functions.iteritems(): |
| namespace[name] = function.invoke |
| |
| def _get_include_path(self, name): |
| """ |
| Resolve the given include def name to a full path. |
| """ |
| |
| # Find the path from the include def name. |
| if not name.startswith('//'): |
| raise ValueError( |
| 'include_defs argument "%s" must begin with //' % name) |
| relative_path = name[2:] |
| return os.path.join(self._project_root, name[2:]) |
| |
| def _include_defs(self, name, implicit_includes=[]): |
| """ |
| Pull the named include into the current caller's context. |
| |
| This method is meant to be installed into the globals of any files or |
| includes that we process. |
| """ |
| |
| # Grab the current build context from the top of the stack. |
| build_env = self._build_env_stack[-1] |
| |
| # Resolve the named include to its path and process it to get its |
| # build context and module. |
| path = self._get_include_path(name) |
| inner_env, mod = self._process_include( |
| path, |
| implicit_includes=implicit_includes) |
| |
| # Look up the caller's stack frame and merge the include's globals |
| # into it's symbol table. |
| frame = inspect.currentframe() |
| while frame.f_globals['__name__'] == __name__: |
| frame = frame.f_back |
| self._merge_globals(mod.__dict__, frame.f_globals) |
| |
| # Pull in the include's accounting of its own referenced includes |
| # into the current build context. |
| build_env.includes.add(path) |
| build_env.includes.update(inner_env.includes) |
| |
| def _push_build_env(self, build_env): |
| """ |
| Set the given build context as the current context. |
| """ |
| |
| self._build_env_stack.append(build_env) |
| self._update_functions(build_env) |
| |
| def _pop_build_env(self): |
| """ |
| Restore the previous build context as the current context. |
| """ |
| |
| self._build_env_stack.pop() |
| if self._build_env_stack: |
| self._update_functions(self._build_env_stack[-1]) |
| |
| def _process(self, build_env, path, implicit_includes=[]): |
| """ |
| Process a build file or include at the given path. |
| """ |
| |
| # First check the cache. |
| cached = self._cache.get(path) |
| if cached is not None: |
| return cached |
| |
| # Install the build context for this input as the current context. |
| self._push_build_env(build_env) |
| |
| # The globals dict that this file will be executed under. |
| default_globals = {} |
| |
| # Install the implicit build functions and adding the 'include_defs' |
| # functions. |
| self._install_functions(default_globals) |
| default_globals['include_defs'] = functools.partial( |
| self._include_defs, |
| implicit_includes=implicit_includes) |
| |
| # If any implicit includes were specified, process them first. |
| for include in implicit_includes: |
| include_path = self._get_include_path(include) |
| inner_env, mod = self._process_include(include_path) |
| self._merge_globals(mod.__dict__, default_globals) |
| build_env.includes.add(include_path) |
| build_env.includes.update(inner_env.includes) |
| |
| # Build a new module for the given file, using the default globals |
| # created above. |
| module = imp.new_module(path) |
| module.__file__ = path |
| module.__dict__.update(default_globals) |
| |
| with open(path) as f: |
| contents = f.read() |
| |
| # Enable absolute imports. This prevents the compiler from trying to |
| # do a relative import first, and warning that this module doesn't |
| # exist in sys.modules. |
| future_features = __future__.absolute_import.compiler_flag |
| code = compile(contents, path, 'exec', future_features, 1) |
| exec(code, module.__dict__) |
| |
| # Restore the previous build context. |
| self._pop_build_env() |
| |
| self._cache[path] = build_env, module |
| return build_env, module |
| |
| def _process_include(self, path, implicit_includes=[]): |
| """ |
| Process the include file at the given path. |
| """ |
| |
| build_env = IncludeContext() |
| return self._process( |
| build_env, |
| path, |
| implicit_includes=implicit_includes) |
| |
| def _process_build_file(self, path, implicit_includes=[]): |
| """ |
| Process the build file at the given path. |
| """ |
| |
| # Create the build file context, including the base path and directory |
| # name of the given path. |
| relative_path_to_build_file = os.path.relpath( |
| path, self._project_root).replace('\\', '/') |
| len_suffix = -len('/' + self._build_file_name) |
| base_path = relative_path_to_build_file[:len_suffix] |
| dirname = os.path.dirname(path) |
| build_env = BuildFileContext(base_path, dirname, self._allow_empty_globs) |
| |
| return self._process( |
| build_env, |
| path, |
| implicit_includes=implicit_includes) |
| |
| def process(self, path): |
| """ |
| Process a build file returning a dict of it's rules and includes. |
| """ |
| |
| build_env, mod = self._process_build_file( |
| os.path.join(self._project_root, path), |
| implicit_includes=self._implicit_includes) |
| values = build_env.rules.values() |
| values.append({"__includes": [path] + sorted(build_env.includes)}) |
| return values |
| |
| |
| # Inexplicably, this script appears to run faster when the arguments passed |
| # into it are absolute paths. However, we want the "buck.base_path" property |
| # of each rule to be printed out to be the base path of the build target that |
| # identifies the rule. That means that when parsing a BUILD file, we must know |
| # its path relative to the root of the project to produce the base path. |
| # |
| # To that end, the first argument to this script must be an absolute path to |
| # the project root. It must be followed by one or more absolute paths to |
| # BUILD files under the project root. If no paths to BUILD files are |
| # specified, then it will traverse the project root for BUILD files, excluding |
| # directories of generated files produced by Buck. |
| # |
| # All of the build rules that are parsed from the BUILD files will be printed |
| # to stdout by a JSON parser. That means that printing out other information |
| # for debugging purposes will likely break the JSON parsing, so be careful! |
| |
| |
| def main(): |
| # Our parent expects to read JSON from our stdout, so if anyone |
| # uses print, buck will complain with a helpful "but I wanted an |
| # array!" message and quit. Redirect stdout to stderr so that |
| # doesn't happen. Actually dup2 the file handle so that writing |
| # to file descriptor 1, os.system, and so on work as expected too. |
| |
| to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'a') |
| os.dup2(sys.stderr.fileno(), sys.stdout.fileno()) |
| |
| parser = optparse.OptionParser() |
| parser.add_option( |
| '--project_root', |
| action='store', |
| type='string', |
| dest='project_root') |
| parser.add_option( |
| '--build_file_name', |
| action='store', |
| type='string', |
| dest="build_file_name") |
| parser.add_option( |
| '--allow_empty_globs', |
| action='store_true', |
| dest='allow_empty_globs', |
| help='Tells the parser not to raise an error when glob returns no results.') |
| parser.add_option( |
| '--include', |
| action='append', |
| dest='include') |
| (options, args) = parser.parse_args() |
| |
| # Even though project_root is absolute path, it may not be concise. For |
| # example, it might be like "C:\project\.\rule". |
| project_root = os.path.abspath(options.project_root) |
| |
| buildFileProcessor = BuildFileProcessor( |
| project_root, |
| options.build_file_name, |
| options.allow_empty_globs, |
| implicit_includes=options.include or []) |
| |
| for build_file in args: |
| values = buildFileProcessor.process(build_file) |
| to_parent.write(json.dumps(values)) |
| to_parent.flush() |
| |
| # "for ... in sys.stdin" in Python 2.x hangs until stdin is closed. |
| for build_file in iter(sys.stdin.readline, ''): |
| values = buildFileProcessor.process(build_file.rstrip()) |
| to_parent.write(json.dumps(values)) |
| to_parent.flush() |
| |
| # Python tries to flush/close stdout when it quits, and if there's a dead |
| # pipe on the other end, it will spit some warnings to stderr. This breaks |
| # tests sometimes. Prevent that by explicitly catching the error. |
| try: |
| to_parent.close() |
| except IOError: |
| pass |
