esp-idf/tools/ci/check_codeowners.py

#!/usr/bin/env python
#
# Utility script for ESP-IDF developers to work with the CODEOWNERS file.
#
# SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0

import argparse
import os
import re
import subprocess
import sys
from typing import List, Optional

from idf_ci_utils import IDF_PATH

CODEOWNERS_PATH = os.path.join(IDF_PATH, '.gitlab', 'CODEOWNERS')
CODEOWNER_GROUP_PREFIX = '@esp-idf-codeowners/'


def get_all_files() -> List[str]:
    """
    Get list of all file paths in the repository.
    """
    # only split on newlines, since file names may contain spaces
    return subprocess.check_output(['git', 'ls-files'], cwd=IDF_PATH).decode('utf-8').strip().split('\n')


def pattern_to_regex(pattern: str) -> str:
    """
    Convert the CODEOWNERS path pattern into a regular expression string.
    """
    orig_pattern = pattern  # for printing errors later

    # Replicates the logic from normalize_pattern function in Gitlab ee/lib/gitlab/code_owners/file.rb:
    if not pattern.startswith('/'):
        pattern = '/**/' + pattern
    if pattern.endswith('/'):
        pattern = pattern + '**/*'

    # Convert the glob pattern into a regular expression:
    # first into intermediate tokens
    pattern = (pattern.replace('**/', ':REGLOB:')
                      .replace('**', ':INVALID:')
                      .replace('*', ':GLOB:')
                      .replace('.', ':DOT:')
                      .replace('?', ':ANY:'))

    if pattern.find(':INVALID:') >= 0:
        raise ValueError("Likely invalid pattern '{}': '**' should be followed by '/'".format(orig_pattern))

    # then into the final regex pattern:
    re_pattern = (pattern.replace(':REGLOB:', '(?:.*/)?')
                         .replace(':GLOB:', '[^/]*')
                         .replace(':DOT:', '[.]')
                         .replace(':ANY:', '.') + '$')
    if re_pattern.startswith('/'):
        re_pattern = '^' + re_pattern

    return re_pattern


def files_by_regex(all_files: List, regex: re.Pattern) -> List:
    """
    Return all files in the repository matching the given regular expresion.
    """
    return [file for file in all_files if regex.search('/' + file)]


def files_by_pattern(all_files: list, pattern: Optional[str]=None) -> List:
    """
    Return all the files in the repository matching the given CODEOWNERS pattern.
    """
    if not pattern:
        return all_files

    return files_by_regex(all_files, re.compile(pattern_to_regex(pattern)))


def action_identify(args: argparse.Namespace) -> None:
    best_match = []
    all_files = get_all_files()
    with open(CODEOWNERS_PATH) as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith('#'):
                continue
            tokens = line.split()
            path_pattern = tokens[0]
            owners = tokens[1:]
            files = files_by_pattern(all_files, path_pattern)
            if args.path in files:
                best_match = owners
    for owner in best_match:
        print(owner)


def action_test_pattern(args: argparse.Namespace) -> None:
    re_pattern = pattern_to_regex(args.pattern)

    if args.regex:
        print(re_pattern)
        return

    files = files_by_regex(get_all_files(), re.compile(re_pattern))
    for f in files:
        print(f)


def action_ci_check(args: argparse.Namespace) -> None:
    errors = []

    def add_error(msg: str) -> None:
        errors.append('{}:{}: {}'.format(CODEOWNERS_PATH, line_no, msg))

    all_files = get_all_files()
    prev_path_pattern = ''
    with open(CODEOWNERS_PATH) as f:
        for line_no, line in enumerate(f, start=1):
            # Skip empty lines and comments
            line = line.strip()
            if line.startswith('# sort-order-reset'):
                prev_path_pattern = ''

            if (not line
                    or line.startswith('#')  # comment
                    or line.startswith('[')  # file group
                    or line.startswith('^[')):  # optional file group
                continue

            # Each line has a form of "<path> <owners>+"
            tokens = line.split()
            path_pattern = tokens[0]
            owners = tokens[1:]
            if not owners:
                add_error('no owners specified for {}'.format(path_pattern))

            # Check that the file is sorted by path patterns
            if not in_order(prev_path_pattern, path_pattern):
                add_error('file is not sorted: {} < {}'.format(path_pattern, prev_path_pattern))
            prev_path_pattern = path_pattern

            # Check that the pattern matches at least one file
            files = files_by_pattern(all_files, path_pattern)
            if not files:
                add_error('no files matched by pattern {}'.format(path_pattern))

            for o in owners:
                # Sanity-check the owner group name
                if not o.startswith(CODEOWNER_GROUP_PREFIX):
                    add_error("owner {} doesn't start with {}".format(o, CODEOWNER_GROUP_PREFIX))

    if not errors:
        print('No errors found.')
    else:
        print('Errors found!')
        for e in errors:
            print(e)
        raise SystemExit(1)


def in_order(prev: str, current: str) -> bool:
    """
    Return True if the ordering is correct for these two lines ('prev' should be before 'current').

    Codeowners should be ordered alphabetically, except that order is also significant for the codeowners
    syntax (the last matching line has priority).

    This means that wildcards are allowed in either order (if wildcard placed first, it's placed before a
    more specific pattern as a catch-all fallback. If wildcard placed second, it's to override the match
    made on a previous line i.e. '/xyz/**/*.py' to override the owner of the Python files inside /xyz/ ).
    """
    if not prev:
        return True  # first element in file

    def is_separator(c: str) -> bool:
        return c in '-_/'  # ignore differences between separators for ordering purposes

    def is_wildcard(c: str) -> bool:
        return c in '?*'

    # looping until we see a different character
    for a,b in zip(prev, current):
        if is_separator(a) and is_separator(b):
            continue
        if is_wildcard(a) or is_wildcard(b):
            return True  # if the strings matched up to one of them having a wildcard, treat as in order
        if a != b:
            return b > a
        assert a == b

    # common substrings up to the common length are the same, so the longer string should be after
    return len(current) >= len(prev)


def main() -> None:
    parser = argparse.ArgumentParser(
        sys.argv[0], description='Internal helper script for working with the CODEOWNERS file.'
    )
    subparsers = parser.add_subparsers(dest='action')

    identify = subparsers.add_parser(
        'identify',
        help='List the owners of the specified path within IDF.'
        "This command doesn't support files inside submodules, or files not added to git repository.",
    )
    identify.add_argument('path', help='Path of the file relative to the root of the repository')

    subparsers.add_parser(
        'ci-check',
        help='Check CODEOWNERS file: every line should match at least one file, sanity-check group names, '
        'check that the file is sorted by paths',
    )

    test_pattern = subparsers.add_parser(
        'test-pattern',
        help='Print files in the repository for a given CODEOWNERS pattern. Useful when adding new rules.'
    )
    test_pattern.add_argument('--regex', action='store_true', help='Print the equivalent regular expression instead of the file list.')
    test_pattern.add_argument('pattern', help='Path pattern to get the list of files for')

    args = parser.parse_args()

    if args.action is None:
        parser.print_help()
        parser.exit(1)

    action_func_name = 'action_' + args.action.replace('-', '_')
    action_func = globals()[action_func_name]
    action_func(args)


if __name__ == '__main__':
    main()