pylint: support multiple pylintrc files per-repo
For a python file, run pylint with the pylintrc file that is found
in the same directory or in any parent directory in the repo.
If no matching pylintrc file can be found then fallback to default
pylintrc file.
Any pylint options specified in PREUPLOAD.cfg will be applied to
all pylint runs.
Bug: 342197112
Test: add pylint trigger to platform/build git, modify python files
and confirm that pylint is executed with correct configuration
Change-Id: I1bda8a491cff5b1f5cfd79b40969210cc417688e
diff --git a/README.md b/README.md
index 9b855ec..23adc8c 100644
--- a/README.md
+++ b/README.md
@@ -319,7 +319,6 @@
# TODO/Limitations
-* `pylint` should support per-directory pylintrc files.
* Some checkers operate on the files as they exist in the filesystem. This is
not easy to fix because the linters require not just the modified file but the
entire repo in order to perform full checks. e.g. `pylint` needs to know what
diff --git a/tools/pylint.py b/tools/pylint.py
index 3fbb148..fc234b4 100755
--- a/tools/pylint.py
+++ b/tools/pylint.py
@@ -21,6 +21,7 @@
import shutil
import sys
import subprocess
+from typing import Dict, List, Optional, Set
assert (sys.version_info.major, sys.version_info.minor) >= (3, 6), (
@@ -62,6 +63,120 @@
return 'pylint'
+def run_lint(pylint: str, unknown: Optional[List[str]],
+ files: Optional[List[str]], init_hook: str,
+ pylintrc: Optional[str] = None) -> bool:
+ """Run lint command.
+
+ Upon error the stdout from pylint will be dumped to stdout and
+ False will be returned.
+ """
+ cmd = [pylint]
+
+ if not files:
+ # No files to analyze for this pylintrc file.
+ return True
+
+ if pylintrc:
+ cmd += ['--rcfile', pylintrc]
+
+ files.sort()
+ cmd += unknown + files
+
+ if init_hook:
+ cmd += ['--init-hook', init_hook]
+
+ try:
+ result = subprocess.run(cmd, stdout=subprocess.PIPE, text=True,
+ check=False)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ print(f'{__file__}: unable to run `{cmd[0]}`: {e}',
+ file=sys.stderr)
+ print(f'{__file__}: Try installing pylint: sudo apt-get install '
+ f'{os.path.basename(cmd[0])}', file=sys.stderr)
+ return False
+
+ raise
+
+ if result.returncode:
+ print(f'{__file__}: Using pylintrc: {pylintrc}')
+ print(result.stdout)
+ return False
+
+ return True
+
+
+def find_parent_dirs_with_pylintrc(leafdir: str,
+ pylintrc_map: Dict[str, Set[str]]) -> None:
+ """Find all dirs containing a pylintrc between root dir and leafdir."""
+
+ # Find all pylintrc files, store the path. The path must end with '/'
+ # to make sure that string compare can be used to compare with full
+ # path to python files later.
+
+ rootdir = os.path.abspath(".") + os.sep
+ key = os.path.abspath(leafdir) + os.sep
+
+ if not key.startswith(rootdir):
+ sys.exit(f'{__file__}: The search directory {key} is outside the '
+ f'repo dir {rootdir}')
+
+ while rootdir != key:
+ # This subdirectory has already been handled, skip it.
+ if key in pylintrc_map:
+ break
+
+ if os.path.exists(os.path.join(key, 'pylintrc')):
+ pylintrc_map.setdefault(key, set())
+ break
+
+ # Go up one directory.
+ key = os.path.join(key, os.pardir) + os.sep
+
+
+def map_pyfiles_to_pylintrc(files: List[str]) -> Dict[str, Set[str]]:
+ """ Map all python files to a pylintrc file.
+
+ Generate dictionary with pylintrc-file dirnames (including trailing /)
+ as key containing sets with corresponding python files.
+ """
+
+ pylintrc_map = {}
+ # We assume pylint is running in the top directory of the project,
+ # so load the pylintrc file from there if it is available.
+ pylintrc = os.path.abspath('pylintrc')
+ if not os.path.exists(pylintrc):
+ pylintrc = DEFAULT_PYLINTRC_PATH
+ # If we pass a non-existent rcfile to pylint, it'll happily ignore
+ # it.
+ assert os.path.exists(pylintrc), f'Could not find {pylintrc}'
+ # Always add top directory, either there is a pylintrc or fallback to
+ # default.
+ key = os.path.abspath('.') + os.sep
+ pylintrc_map[key] = set()
+
+ search_dirs = {os.path.dirname(x) for x in files}
+ for search_dir in search_dirs:
+ find_parent_dirs_with_pylintrc(search_dir, pylintrc_map)
+
+ # List of directories where pylintrc files are stored, most
+ # specific path first.
+ rc_dir_names = sorted(pylintrc_map, reverse=True)
+ # Map all python files to a pylintrc file.
+ for f in files:
+ f_full = os.path.abspath(f)
+ for rc_dir in rc_dir_names:
+ # The pylintrc map keys always have trailing /.
+ if f_full.startswith(rc_dir):
+ pylintrc_map[rc_dir].add(f)
+ break
+ else:
+ sys.exit(f'{__file__}: Failed to map file {f} to a pylintrc file.')
+
+ return pylintrc_map
+
+
def get_parser():
"""Return a command line parser."""
parser = argparse.ArgumentParser(description=__doc__)
@@ -70,7 +185,7 @@
help='Force Python 3 mode')
parser.add_argument('--executable-path',
help='The path of the pylint executable.')
- parser.add_argument('--no-rcfile',
+ parser.add_argument('--no-rcfile', dest='use_default_conf',
help='Specify to use the executable\'s default '
'configuration.',
action='store_true')
@@ -82,6 +197,7 @@
"""The main entry."""
parser = get_parser()
opts, unknown = parser.parse_known_args(argv)
+ ret = 0
pylint = opts.executable_path
if pylint is None:
@@ -94,35 +210,25 @@
if opts.py3:
is_pylint3(pylint)
- cmd = [pylint]
- if not opts.no_rcfile:
- # We assume pylint is running in the top directory of the project,
- # so load the pylintrc file from there if it's available.
- pylintrc = os.path.abspath('pylintrc')
- if not os.path.exists(pylintrc):
- pylintrc = DEFAULT_PYLINTRC_PATH
- # If we pass a non-existent rcfile to pylint, it'll happily ignore
- # it.
- assert os.path.exists(pylintrc), f'Could not find {pylintrc}'
- cmd += ['--rcfile', pylintrc]
+ if not opts.use_default_conf:
+ pylintrc_map = map_pyfiles_to_pylintrc(opts.files)
+ first = True
+ for rc_dir, files in sorted(pylintrc_map.items()):
+ pylintrc = os.path.join(rc_dir, 'pylintrc')
+ if first:
+ first = False
+ assert os.path.abspath(rc_dir) == os.path.abspath('.'), (
+ f'{__file__}: pylintrc in top dir not first in list')
+ if not os.path.exists(pylintrc):
+ pylintrc = DEFAULT_PYLINTRC_PATH
+ if not run_lint(pylint, unknown, sorted(files),
+ opts.init_hook, pylintrc):
+ ret = 1
+ # Not using rc files, pylint default behaviour.
+ elif not run_lint(pylint, unknown, sorted(opts.files), opts.init_hook):
+ ret = 1
- cmd += unknown + opts.files
-
- if opts.init_hook:
- cmd += ['--init-hook', opts.init_hook]
-
- try:
- os.execvp(cmd[0], cmd)
- return 0
- except OSError as e:
- if e.errno == errno.ENOENT:
- print(f'{__file__}: unable to run `{cmd[0]}`: {e}',
- file=sys.stderr)
- print(f'{__file__}: Try installing pylint: sudo apt-get install '
- f'{os.path.basename(cmd[0])}', file=sys.stderr)
- return 1
-
- raise
+ return ret
if __name__ == '__main__':
