esp-idf/tools/ci/idf_ci_utils.py

# internal use only for CI
# some CI related util functions
#
# SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import functools
import logging
import os
import re
import subprocess
import sys
from typing import List, Optional

IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')))


def get_submodule_dirs(full_path: bool = False) -> List:
    """
    To avoid issue could be introduced by multi-os or additional dependency,
    we use python and git to get this output
    :return: List of submodule dirs
    """
    dirs = []
    try:
        lines = subprocess.check_output(
            ['git', 'config', '--file', os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')),
             '--get-regexp', 'path']).decode('utf8').strip().split('\n')
        for line in lines:
            _, path = line.split(' ')
            if full_path:
                dirs.append(os.path.join(IDF_PATH, path))
            else:
                dirs.append(path)
    except Exception as e:  # pylint: disable=W0703
        logging.warning(str(e))

    return dirs


def _check_git_filemode(full_path):  # type: (str) -> bool
    try:
        stdout = subprocess.check_output(['git', 'ls-files', '--stage', full_path]).strip().decode('utf-8')
    except subprocess.CalledProcessError:
        return True

    mode = stdout.split(' ', 1)[0]  # e.g. 100644 for a rw-r--r--
    if any([int(i, 8) & 1 for i in mode[-3:]]):
        return True
    return False


def is_executable(full_path: str) -> bool:
    """
    os.X_OK will always return true on windows. Use git to check file mode.
    :param full_path: file full path
    :return: True is it's an executable file
    """
    if sys.platform == 'win32':
        return _check_git_filemode(full_path)
    return os.access(full_path, os.X_OK)


def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]:
    """
    Get the result of git ls-files
    :param path: path to run git ls-files
    :param full_path: return full path if set to True
    :return: list of file paths
    """
    try:
        # this is a workaround when using under worktree
        # if you're using worktree, when running git commit a new environment variable GIT_DIR would be declared,
        # the value should be <origin_repo_path>/.git/worktrees/<worktree name>
        # This would effect the return value of `git ls-files`, unset this would use the `cwd`value or its parent
        # folder if no `.git` folder found in `cwd`.
        workaround_env = os.environ.copy()
        workaround_env.pop('GIT_DIR', None)
        files = subprocess.check_output(['git', 'ls-files'], cwd=path, env=workaround_env) \
            .decode('utf8').strip().split('\n')
    except Exception as e:  # pylint: disable=W0703
        logging.warning(str(e))
        files = []
    return [os.path.join(path, f) for f in files] if full_path else files


# this function is a commit from
# https://github.com/python/cpython/pull/6299/commits/bfd63120c18bd055defb338c075550f975e3bec1
# In order to solve python https://bugs.python.org/issue9584
# glob pattern does not support brace expansion issue
def _translate(pat: str) -> str:
    """Translate a shell PATTERN to a regular expression.
    There is no way to quote meta-characters.
    """
    i, n = 0, len(pat)
    res = ''
    while i < n:
        c = pat[i]
        i = i + 1
        if c == '*':
            res = res + '.*'
        elif c == '?':
            res = res + '.'
        elif c == '[':
            j = i
            if j < n and pat[j] == '!':
                j = j + 1
            if j < n and pat[j] == ']':
                j = j + 1
            while j < n and pat[j] != ']':
                j = j + 1
            if j >= n:
                res = res + '\\['
            else:
                stuff = pat[i:j]
                if '--' not in stuff:
                    stuff = stuff.replace('\\', r'\\')
                else:
                    chunks = []
                    k = i + 2 if pat[i] == '!' else i + 1
                    while True:
                        k = pat.find('-', k, j)
                        if k < 0:
                            break
                        chunks.append(pat[i:k])
                        i = k + 1
                        k = k + 3
                    chunks.append(pat[i:j])
                    # Escape backslashes and hyphens for set difference (--).
                    # Hyphens that create ranges shouldn't be escaped.
                    stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-')
                                     for s in chunks)
                # Escape set operations (&&, ~~ and ||).
                stuff = re.sub(r'([&~|])', r'\\\1', stuff)
                i = j + 1
                if stuff[0] == '!':
                    stuff = '^' + stuff[1:]
                elif stuff[0] in ('^', '['):
                    stuff = '\\' + stuff
                res = '%s[%s]' % (res, stuff)
        elif c == '{':
            # Handling of brace expression: '{PATTERN,PATTERN,...}'
            j = 1
            while j < n and pat[j] != '}':
                j = j + 1
            if j >= n:
                res = res + '\\{'
            else:
                stuff = pat[i:j]
                i = j + 1

                # Find indices of ',' in pattern excluding r'\,'.
                # E.g. for r'a\,a,b\b,c' it will be [4, 8]
                indices = [m.end() for m in re.finditer(r'[^\\],', stuff)]

                # Splitting pattern string based on ',' character.
                # Also '\,' is translated to ','. E.g. for r'a\,a,b\b,c':
                # * first_part = 'a,a'
                # * last_part = 'c'
                # * middle_part = ['b,b']
                first_part = stuff[:indices[0] - 1].replace(r'\,', ',')
                last_part = stuff[indices[-1]:].replace(r'\,', ',')
                middle_parts = [
                    stuff[st:en - 1].replace(r'\,', ',')
                    for st, en in zip(indices, indices[1:])
                ]

                # creating the regex from splitted pattern. Each part is
                # recursivelly evaluated.
                expanded = functools.reduce(
                    lambda a, b: '|'.join((a, b)),
                    (_translate(elem) for elem in [first_part] + middle_parts + [last_part])
                )
                res = '%s(%s)' % (res, expanded)
        else:
            res = res + re.escape(c)
    return res


def translate(pat: str) -> str:
    res = _translate(pat)
    return r'(?s:%s)\Z' % res


magic_check = re.compile('([*?[{])')
magic_check_bytes = re.compile(b'([*?[{])')
# cpython github PR 6299 ends here

# Here's the code block we're going to use to monkey patch ``glob`` module and ``fnmatch`` modules
# DO NOT monkey patch here, only patch where you really needs
#
# import glob
# import fnmatch
# from idf_ci_utils import magic_check, magic_check_bytes, translate
# glob.magic_check = magic_check
# glob.magic_check_bytes = magic_check_bytes
# fnmatch.translate = translate


def is_in_directory(file_path: str, folder: str) -> bool:
    return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep)


def get_pytest_dirs(folder: str, under_dir: Optional[str] = None) -> List[str]:
    from io import StringIO

    import pytest
    from _pytest.nodes import Item

    class CollectPlugin:
        def __init__(self) -> None:
            self.nodes: List[Item] = []

        def pytest_collection_modifyitems(self, items: List[Item]) -> None:
            for item in items:
                self.nodes.append(item)

    collector = CollectPlugin()

    sys_stdout = sys.stdout
    sys.stdout = StringIO()  # swallow the output
    pytest.main(['--collect-only', folder], plugins=[collector])
    sys.stdout = sys_stdout  # restore sys.stdout

    test_file_paths = set(node.fspath for node in collector.nodes)

    if under_dir:
        return [os.path.dirname(file) for file in test_file_paths if is_in_directory(file, under_dir)]

    return [os.path.dirname(file) for file in test_file_paths]