esp-idf/tools/ci/check_codeowners.py

#!/usr/bin/env python
#
# Utility script for ESP-IDF developers to work with the CODEOWNERS file.
#
# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import re
import subprocess
import sys

from idf_ci_utils import IDF_PATH

CODEOWNERS_PATH = os.path.join(IDF_PATH, '.gitlab', 'CODEOWNERS')
CODEOWNER_GROUP_PREFIX = '@esp-idf-codeowners/'


def get_all_files():
    """
    Get list of all file paths in the repository.
    """
    # only split on newlines, since file names may contain spaces
    return subprocess.check_output(['git', 'ls-files'], cwd=IDF_PATH).decode('utf-8').strip().split('\n')


def pattern_to_regex(pattern):
    """
    Convert the CODEOWNERS path pattern into a regular expression string.
    """
    orig_pattern = pattern  # for printing errors later

    # Replicates the logic from normalize_pattern function in Gitlab ee/lib/gitlab/code_owners/file.rb:
    if not pattern.startswith('/'):
        pattern = '/**/' + pattern
    if pattern.endswith('/'):
        pattern = pattern + '**/*'

    # Convert the glob pattern into a regular expression:
    # first into intermediate tokens
    pattern = (pattern.replace('**/', ':REGLOB:')
                      .replace('**', ':INVALID:')
                      .replace('*', ':GLOB:')
                      .replace('.', ':DOT:')
                      .replace('?', ':ANY:'))

    if pattern.find(':INVALID:') >= 0:
        raise ValueError("Likely invalid pattern '{}': '**' should be followed by '/'".format(orig_pattern))

    # then into the final regex pattern:
    re_pattern = (pattern.replace(':REGLOB:', '(?:.*/)?')
                         .replace(':GLOB:', '[^/]*')
                         .replace(':DOT:', '[.]')
                         .replace(':ANY:', '.') + '$')
    if re_pattern.startswith('/'):
        re_pattern = '^' + re_pattern

    return re_pattern


def files_by_regex(all_files, regex):
    """
    Return all files in the repository matching the given regular expresion.
    """
    return [file for file in all_files if regex.search('/' + file)]


def files_by_pattern(all_files, pattern=None):
    """
    Return all the files in the repository matching the given CODEOWNERS pattern.
    """
    if not pattern:
        return all_files

    return files_by_regex(all_files, re.compile(pattern_to_regex(pattern)))


def action_identify(args):
    best_match = []
    all_files = get_all_files()
    with open(CODEOWNERS_PATH) as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith('#'):
                continue
            tokens = line.split()
            path_pattern = tokens[0]
            owners = tokens[1:]
            files = files_by_pattern(all_files, path_pattern)
            if args.path in files:
                best_match = owners
    for owner in best_match:
        print(owner)


def action_test_pattern(args):
    re_pattern = pattern_to_regex(args.pattern)

    if args.regex:
        print(re_pattern)
        return

    files = files_by_regex(get_all_files(), re.compile(re_pattern))
    for f in files:
        print(f)


def action_ci_check(args):
    errors = []

    def add_error(msg):
        errors.append('{}:{}: {}'.format(CODEOWNERS_PATH, line_no, msg))

    all_files = get_all_files()
    prev_path_pattern = ''
    with open(CODEOWNERS_PATH) as f:
        for line_no, line in enumerate(f, start=1):
            # Skip empty lines and comments
            line = line.strip()
            if line.startswith('# sort-order-reset'):
                prev_path_pattern = ''

            if not line or line.startswith('#'):
                continue

            # Each line has a form of "<path> <owners>+"
            tokens = line.split()
            path_pattern = tokens[0]
            owners = tokens[1:]
            if not owners:
                add_error('no owners specified for {}'.format(path_pattern))

            # Check that the file is sorted by path patterns
            path_pattern_for_cmp = path_pattern.replace('-', '_')  # ignore difference between _ and - for ordering
            if prev_path_pattern and path_pattern_for_cmp < prev_path_pattern:
                add_error('file is not sorted: {} < {}'.format(path_pattern_for_cmp, prev_path_pattern))
            prev_path_pattern = path_pattern_for_cmp

            # Check that the pattern matches at least one file
            files = files_by_pattern(all_files, path_pattern)
            if not files:
                add_error('no files matched by pattern {}'.format(path_pattern))

            for o in owners:
                # Sanity-check the owner group name
                if not o.startswith(CODEOWNER_GROUP_PREFIX):
                    add_error("owner {} doesn't start with {}".format(o, CODEOWNER_GROUP_PREFIX))

    if not errors:
        print('No errors found.')
    else:
        print('Errors found!')
        for e in errors:
            print(e)
        raise SystemExit(1)


def main():
    parser = argparse.ArgumentParser(
        sys.argv[0], description='Internal helper script for working with the CODEOWNERS file.'
    )
    subparsers = parser.add_subparsers(dest='action')

    identify = subparsers.add_parser(
        'identify',
        help='List the owners of the specified path within IDF.'
        "This command doesn't support files inside submodules, or files not added to git repository.",
    )
    identify.add_argument('path', help='Path of the file relative to the root of the repository')

    subparsers.add_parser(
        'ci-check',
        help='Check CODEOWNERS file: every line should match at least one file, sanity-check group names, '
        'check that the file is sorted by paths',
    )

    test_pattern = subparsers.add_parser(
        'test-pattern',
        help='Print files in the repository for a given CODEOWNERS pattern. Useful when adding new rules.'
    )
    test_pattern.add_argument('--regex', action='store_true', help='Print the equivalent regular expression instead of the file list.')
    test_pattern.add_argument('pattern', help='Path pattern to get the list of files for')

    args = parser.parse_args()

    if args.action is None:
        parser.print_help()
        parser.exit(1)

    action_func_name = 'action_' + args.action.replace('-', '_')
    action_func = globals()[action_func_name]
    action_func(args)


if __name__ == '__main__':
    main()