| // Copyright (C) 2017 The Android Open Source Project |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.googlesource.gerrit.plugins.findowners; |
| |
| import com.google.common.flogger.FluentLogger; |
| import com.google.gerrit.server.git.GitRepositoryManager; |
| import com.google.gerrit.server.permissions.PermissionBackend; |
| import java.io.IOException; |
| import java.util.ArrayDeque; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Deque; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| |
| /** |
| * Parse lines in an OWNERS file and put them into an OwnersDb. One Parser object should be created |
| * to parse only one OWNERS file. It keeps permissionBackend, readFiles, repoManager, project, |
| * branch, and filePath of the OWNERS file so it can find files that are included by OWNERS. |
| * |
| * <p>The usage pattern is: |
| * |
| * <pre> |
| * Parser parser = new Parser(permissionBackend, readFiles, repoManager, project, branch, repoFilePath); |
| * String content = OwnersDb.getRepoFile(permissionBackend, readFiles, repoManager, null, null, |
| * project, branch, repoFilePath, logs); |
| * Parser.Result result = parser.parseFile(dirPath, content); |
| * </pre> |
| * |
| * <p>OWNERS file syntax, semantics, and examples are included in syntax.md. |
| */ |
| class Parser { |
| private static final FluentLogger logger = FluentLogger.forEnclosingClass(); |
| |
| // Artifical owner token for "set noparent" when used in per-file. |
| protected static final String TOK_SET_NOPARENT = "set noparent"; |
| |
| // Globs and emails are separated by commas with optional spaces around a comma. |
| protected static final String COMMA = "[\\s]*,[\\s]*"; // used in unit tests |
| |
| // Separator for project and file paths in an include line. |
| private static final String COLON = "[\\s]*:[\\s]*"; // project:file |
| |
| private static final String BOL = "^[\\s]*"; // begin-of-line |
| private static final String EOL = "[\\s]*(#.*)?$"; // end-of-line |
| private static final String GLOB = "[^\\s,=]+"; // a file glob |
| |
| // TODO: have a more precise email address pattern. |
| private static final String EMAIL_OR_STAR = "([^\\s<>@,]+@[^\\s<>@#,]+|\\*)"; |
| private static final String EMAIL_LIST = |
| "(" + EMAIL_OR_STAR + "(" + COMMA + EMAIL_OR_STAR + ")*)"; |
| |
| // A Gerrit project name followed by a colon and optional spaces. |
| private static final String PROJECT_NAME = "([^\\s:]+" + COLON + ")?"; |
| |
| // A relative or absolute file path name without any colon or space character. |
| private static final String FILE_PATH = "([^\\s:#]+)"; |
| |
| private static final String PROJECT_AND_FILE = PROJECT_NAME + FILE_PATH; |
| |
| private static final String SET_NOPARENT = "set[\\s]+noparent"; |
| |
| private static final String FILE_DIRECTIVE = "file:[\\s]*" + PROJECT_AND_FILE; |
| private static final String INCLUDE_OR_FILE = "(file:[\\s]*|include[\\s]+)"; |
| |
| // Simple input lines with 0 or 1 sub-pattern. |
| private static final Pattern PAT_COMMENT = Pattern.compile(BOL + EOL); |
| private static final Pattern PAT_EMAIL = Pattern.compile(BOL + EMAIL_OR_STAR + EOL); |
| private static final Pattern PAT_INCLUDE = |
| Pattern.compile(BOL + INCLUDE_OR_FILE + PROJECT_AND_FILE + EOL); |
| private static final Pattern PAT_NO_PARENT = Pattern.compile(BOL + SET_NOPARENT + EOL); |
| |
| // Patterns to match trimmed globs, emails, and set noparent in per-file lines. |
| private static final Pattern PAT_PER_FILE_OWNERS = |
| Pattern.compile("^(" + EMAIL_LIST + "|" + SET_NOPARENT + "|" + FILE_DIRECTIVE + ")$"); |
| private static final Pattern PAT_GLOBS = |
| Pattern.compile("^(" + GLOB + "(" + COMMA + GLOB + ")*)$"); |
| // PAT_PER_FILE matches a line to two groups: (1) globs, (2) emails |
| // Trimmed 1st group should match PAT_GLOBS; |
| // trimmed 2nd group should match PAT_PER_FILE_OWNERS. |
| private static final Pattern PAT_PER_FILE = |
| Pattern.compile(BOL + "per-file[\\s]+([^=#]+)=[\\s]*([^#]+)" + EOL); |
| // Fetch the include/file part of a line with correct syntax. |
| private static final Pattern PAT_INCLUDE_OR_FILE = |
| Pattern.compile("^.*(" + INCLUDE_OR_FILE + PROJECT_AND_FILE + ")" + EOL); |
| |
| // A parser keeps current permissionBackend, readFiles, repoManager, project, branch, |
| // included file path, and debug/trace logs. |
| private final PermissionBackend permissionBackend; |
| private final Map<String, String> readFiles; |
| private final GitRepositoryManager repoManager; |
| private final String branch; // All owners files are read from the same branch. |
| private final IncludeStack stack; // a stack of including files. |
| private final List<String> logs; // Keeps debug/trace messages. |
| private final Map<String, Result> savedResults; // projectName:filePath => Parser.Result |
| |
| static class IncludeStack { |
| Deque<String> projectName; // project/repository name of included file |
| Deque<String> filePath; // absolute or relative path of included file |
| Set<String> allFiles; // to detect recursive inclusion quickly |
| |
| IncludeStack(String project, String file) { |
| projectName = new ArrayDeque<>(); |
| filePath = new ArrayDeque<>(); |
| allFiles = new HashSet<>(); |
| push(project, file); |
| } |
| |
| void push(String project, String file) { |
| projectName.push(project); |
| filePath.push(file); |
| allFiles.add(getFileKey(project, file)); |
| } |
| |
| void pop() { |
| allFiles.remove(getFileKey(currentProject(), currentFile())); |
| projectName.pop(); |
| filePath.pop(); |
| } |
| |
| String currentProject() { |
| return projectName.peek(); |
| } |
| |
| String currentFile() { |
| return filePath.peek(); |
| } |
| |
| boolean contains(String project, String file) { |
| return allFiles.contains(getFileKey(project, file)); |
| } |
| } |
| |
| // For simple unit tests without a repository. |
| Parser(String project, String branch, String file) { |
| this(null, null, null, project, branch, file, new ArrayList<>()); |
| } |
| |
| Parser( |
| PermissionBackend permissionBackend, |
| Map<String, String> readFiles, |
| GitRepositoryManager repoManager, |
| String project, |
| String branch, |
| String file) { |
| this(permissionBackend, readFiles, repoManager, project, branch, file, new ArrayList<>()); |
| } |
| |
| Parser( |
| PermissionBackend permissionBackend, |
| Map<String, String> readFiles, |
| GitRepositoryManager repoManager, |
| String project, |
| String branch, |
| String file, |
| List<String> logs) { |
| this.permissionBackend = permissionBackend; |
| this.readFiles = readFiles; |
| this.repoManager = repoManager; |
| this.branch = branch; |
| this.logs = logs; |
| stack = new IncludeStack(project, normalizedRepoDirFilePath(".", file)); |
| savedResults = new HashMap<>(); |
| } |
| |
| static boolean isComment(String line) { |
| return PAT_COMMENT.matcher(line).matches(); |
| } |
| |
| static boolean isInclude(String line) { |
| return PAT_INCLUDE.matcher(line).matches(); |
| } |
| |
| static boolean isGlobs(String line) { |
| return PAT_GLOBS.matcher(line).matches(); |
| } |
| |
| static boolean isNoParent(String line) { |
| return PAT_NO_PARENT.matcher(line).matches(); |
| } |
| |
| static String parseEmail(String line) { |
| Matcher m = Parser.PAT_EMAIL.matcher(line); |
| return m.matches() ? m.group(1).trim() : null; |
| } |
| |
| static String[] parseInclude(String project, String line) { |
| Matcher m = Parser.PAT_INCLUDE.matcher(line); |
| if (!m.matches()) { |
| return null; |
| } |
| String keyword = m.group(1).trim(); |
| if (keyword.equals("file:")) { |
| keyword = "file"; |
| } |
| String projectName = m.group(2); |
| if (projectName != null && projectName.length() > 1) { |
| // PROJECT_NAME ends with ':' |
| projectName = projectName.split(COLON, -1)[0].trim(); |
| } else { |
| projectName = project; // default project name |
| } |
| return new String[] {keyword, projectName, m.group(3).trim()}; |
| } |
| |
| static String removeExtraSpaces(String s) { |
| return s.trim().replaceAll("[\\s]+", " ").replaceAll("[\\s]*:[\\s]*", ":"); |
| } |
| |
| static String[] parsePerFile(String line) { |
| Matcher m = PAT_PER_FILE.matcher(line); |
| if (!m.matches() |
| || !isGlobs(m.group(1).trim()) |
| || !PAT_PER_FILE_OWNERS.matcher(m.group(2).trim()).matches()) { |
| return null; |
| } |
| return new String[] {removeExtraSpaces(m.group(1)), removeExtraSpaces(m.group(2))}; |
| } |
| |
| static String[] parsePerFileOwners(String line) { |
| String[] globsAndOwners = parsePerFile(line); |
| return (globsAndOwners != null) ? globsAndOwners[1].split(COMMA, -1) : null; |
| } |
| |
| static String getIncludeOrFile(String line) { |
| Matcher m = PAT_INCLUDE_OR_FILE.matcher(line); |
| return m.matches() ? removeExtraSpaces(m.group(1)) : ""; |
| } |
| |
| static class Result { |
| boolean stopLooking; // if this file contains set noparent |
| Set<String> warnings; // unique warning messages |
| Set<String> errors; // unique error messages |
| Map<String, Set<String>> owner2paths; // maps from owner email to pathGlobs |
| Set<String> noParentGlobs; // per-file dirpath+glob with "set noparent" |
| |
| Result() { |
| stopLooking = false; |
| warnings = new HashSet<>(); |
| errors = new HashSet<>(); |
| owner2paths = new HashMap<>(); |
| noParentGlobs = new HashSet<>(); |
| } |
| |
| void append(Result r, String dir, boolean addAll) { |
| // addAll is true when the Result is from an include statements. |
| // It is false for the included result of "file:" directive, which |
| // only collects owner emails, not per-file or set noparent statement. |
| warnings.addAll(r.warnings); |
| errors.addAll(r.errors); |
| if (addAll) { |
| stopLooking = stopLooking || r.stopLooking; |
| for (String glob : r.noParentGlobs) { |
| noParentGlobs.add(dir + glob); |
| } |
| } |
| for (String key : r.owner2paths.keySet()) { |
| for (String path : r.owner2paths.get(key)) { |
| // In an included file, top-level owener emails have empty dir path. |
| if (path.isEmpty() || addAll) { |
| Util.addToMap(owner2paths, key, dir + path); |
| } |
| } |
| } |
| } |
| } |
| |
| /** |
| * Parse given lines of an OWNERS files; return parsed Result. It can recursively call itself to |
| * parse included files. |
| * |
| * @param dir is the directory that contains "changed files" of a CL, not necessarily the OWNERS |
| * or included file directory. "owners" found in lines control changed files in 'dir'. 'dir' |
| * ends with '/' or is empty when parsing an included file. |
| * @param lines are the source lines of the file to be parsed. |
| * @return the parsed data |
| */ |
| Result parseFile(String dir, String[] lines) { |
| Result result = new Result(); |
| int n = 0; |
| for (String line : lines) { |
| parseLine(result, dir, line, ++n); |
| } |
| return result; |
| } |
| |
| Result parseFile(String dir, String content) { |
| return parseFile(dir, content.split("\\R")); |
| } |
| |
| private String normalizedRepoDirFilePath(String dir, String path) { |
| try { |
| return Util.normalizedRepoDirFilePath(dir, path); |
| } catch (IOException e) { |
| String msg = "Fail to normalized path " + dir + " / " + path; |
| logger.atSevere().withCause(e).log(msg); |
| logs.add(msg + ":" + e.getMessage()); |
| return dir + "/" + path; |
| } |
| } |
| |
| /** |
| * Parse a line in OWNERS file and add parsed info into result. This function should be called |
| * only by parseFile and Parser unit tests. |
| * |
| * @param result a Result object to keep parsed info. |
| * @param dir the path to OWNERS file directory. |
| * @param line the source line. |
| * @param num the line number. |
| */ |
| void parseLine(Result result, String dir, String line, int num) { |
| String email; |
| String[] globsAndOwners; |
| String[] parsedKPF; // parsed keyword, projectName, filePath |
| if (isNoParent(line)) { |
| result.stopLooking = true; |
| } else if (isComment(line)) { |
| // ignore comment and empty lines. |
| } else if ((email = parseEmail(line)) != null) { |
| Util.addToMap(result.owner2paths, email, dir); // here dir is not a glob |
| } else if ((globsAndOwners = parsePerFile(line)) != null) { |
| String[] dirGlobs = globsAndOwners[0].split(COMMA, -1); |
| String directive = globsAndOwners[1]; |
| if (directive.equals(Parser.TOK_SET_NOPARENT)) { |
| // per-file globs = set noparent |
| for (String glob : dirGlobs) { |
| result.noParentGlobs.add(dir + glob); |
| } |
| } else { |
| List<String> ownerEmails; |
| if ((parsedKPF = parseInclude(stack.currentProject(), directive)) == null) { |
| // per-file globs = ownerEmails |
| ownerEmails = Arrays.asList(directive.split(COMMA, -1)); |
| } else { |
| // per-file globs = file: projectFile |
| ownerEmails = new ArrayList<>(); |
| Result r = new Result(); |
| includeFile(r, "", num, parsedKPF, false); |
| for (String key : r.owner2paths.keySet()) { |
| for (String path : r.owner2paths.get(key)) { |
| if (path.isEmpty()) { |
| ownerEmails.add(key); |
| break; |
| } |
| } |
| } |
| } |
| for (String glob : dirGlobs) { |
| for (String owner : ownerEmails) { |
| Util.addToMap(result.owner2paths, owner, dir + glob); |
| } |
| } |
| } |
| } else if ((parsedKPF = parseInclude(stack.currentProject(), line)) != null) { |
| includeFile(result, dir, num, parsedKPF, parsedKPF[0].equals("include")); |
| } else { |
| result.errors.add(errorMsg(stack.currentFile(), num, "ignored unknown line", line)); |
| } |
| } |
| |
| /** |
| * Find and parse an included file and append data to the 'result'. For an 'include' statement, |
| * parsed data is all appended to the given result parameter. For a 'file:' statement or |
| * directive, only owner emails are appended. If the project+file name is found in the stored |
| * result set, the stored result is reused. The inclusion is skipped if the to be included file is |
| * already on the including file stack. |
| * |
| * @param result to where the included file data should be added. |
| * @param dir the including file's directory or glob. |
| * @param num source code line number |
| * @param parsedKPF the parsed line of include or file directive. |
| * @param addAll to add all parsed data into result or not. |
| */ |
| private void includeFile(Result result, String dir, int num, String[] parsedKPF, boolean addAll) { |
| String keyword = parsedKPF[0]; |
| String project = parsedKPF[1]; |
| String file = parsedKPF[2]; |
| String includeKPF = keyword + ":" + getFileKey(project, file); |
| // Like C/C++ #include, when f1 includes f2 with a relative file path, |
| // use f1's directory, not 'dir', as the base for relative path. |
| // 'dir' is the directory of OWNERS file, which might include f1 indirectly. |
| String repoFile = normalizedRepoDirFilePath(Util.getParentDir(stack.currentFile()), file); |
| if (stack.contains(project, repoFile)) { |
| logs.add("parseLine:errorRecursion:" + includeKPF); |
| result.errors.add(errorMsg(stack.currentFile(), num, "recursive include", includeKPF)); |
| return; |
| } |
| String savedResultKey = getFileKey(project, repoFile); |
| Result includedFileResult = savedResults.get(savedResultKey); |
| if (null != includedFileResult) { |
| logs.add("parseLine:useSaved:" + includeKPF); |
| } else { |
| stack.push(project, repoFile); |
| logs.add("parseLine:" + includeKPF); |
| String content = |
| OwnersDb.getRepoFile( |
| permissionBackend, |
| readFiles, |
| repoManager, |
| null, |
| null, |
| project, |
| branch, |
| repoFile, |
| logs); |
| if (content != null && !content.isEmpty()) { |
| includedFileResult = parseFile("", content); |
| } else { |
| logs.add("parseLine:" + keyword + ":()"); |
| includedFileResult = new Result(); |
| } |
| stack.pop(); |
| savedResults.put(savedResultKey, includedFileResult); |
| } |
| result.append(includedFileResult, dir, addAll); |
| } |
| |
| // Build a readable key or output string for a (project, file) pair. |
| static String getFileKey(String project, String file) { |
| return project + ":" + file; |
| } |
| |
| // Build a readable key or output string for a (project, branch, file) tuple. |
| static String getFileKey(String project, String branch, String file) { |
| return project + ":" + branch + ":" + file; |
| } |
| |
| private static String createMsgLine(String prefix, String file, int n, String msg, String line) { |
| return prefix + file + ":" + n + ": " + msg + ": [" + line + "]"; |
| } |
| |
| static String errorMsg(String file, int n, String msg, String line) { |
| return createMsgLine("Error: ", file, n, msg, line); |
| } |
| |
| static String warningMsg(String file, int n, String msg, String line) { |
| return createMsgLine("Warning: ", file, n, msg, line); |
| } |
| } |