idf_ci_utils.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. # internal use only for CI
  2. # some CI related util functions
  3. #
  4. # SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
  5. # SPDX-License-Identifier: Apache-2.0
  6. #
  7. import io
  8. import logging
  9. import os
  10. import subprocess
  11. import sys
  12. from contextlib import redirect_stdout
  13. from typing import TYPE_CHECKING, List
  14. if TYPE_CHECKING:
  15. from _pytest.nodes import Function
  16. IDF_PATH = os.path.abspath(
  17. os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))
  18. )
  19. def get_submodule_dirs(full_path: bool = False) -> List:
  20. """
  21. To avoid issue could be introduced by multi-os or additional dependency,
  22. we use python and git to get this output
  23. :return: List of submodule dirs
  24. """
  25. dirs = []
  26. try:
  27. lines = (
  28. subprocess.check_output(
  29. [
  30. 'git',
  31. 'config',
  32. '--file',
  33. os.path.realpath(os.path.join(IDF_PATH, '.gitmodules')),
  34. '--get-regexp',
  35. 'path',
  36. ]
  37. )
  38. .decode('utf8')
  39. .strip()
  40. .split('\n')
  41. )
  42. for line in lines:
  43. _, path = line.split(' ')
  44. if full_path:
  45. dirs.append(os.path.join(IDF_PATH, path))
  46. else:
  47. dirs.append(path)
  48. except Exception as e: # pylint: disable=W0703
  49. logging.warning(str(e))
  50. return dirs
  51. def _check_git_filemode(full_path): # type: (str) -> bool
  52. try:
  53. stdout = (
  54. subprocess.check_output(['git', 'ls-files', '--stage', full_path])
  55. .strip()
  56. .decode('utf-8')
  57. )
  58. except subprocess.CalledProcessError:
  59. return True
  60. mode = stdout.split(' ', 1)[0] # e.g. 100644 for a rw-r--r--
  61. if any([int(i, 8) & 1 for i in mode[-3:]]):
  62. return True
  63. return False
  64. def is_executable(full_path: str) -> bool:
  65. """
  66. os.X_OK will always return true on windows. Use git to check file mode.
  67. :param full_path: file full path
  68. :return: True is it's an executable file
  69. """
  70. if sys.platform == 'win32':
  71. return _check_git_filemode(full_path)
  72. return os.access(full_path, os.X_OK)
  73. def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]:
  74. """
  75. Get the result of git ls-files
  76. :param path: path to run git ls-files
  77. :param full_path: return full path if set to True
  78. :return: list of file paths
  79. """
  80. try:
  81. # this is a workaround when using under worktree
  82. # if you're using worktree, when running git commit a new environment variable GIT_DIR would be declared,
  83. # the value should be <origin_repo_path>/.git/worktrees/<worktree name>
  84. # This would effect the return value of `git ls-files`, unset this would use the `cwd`value or its parent
  85. # folder if no `.git` folder found in `cwd`.
  86. workaround_env = os.environ.copy()
  87. workaround_env.pop('GIT_DIR', None)
  88. files = (
  89. subprocess.check_output(['git', 'ls-files'], cwd=path, env=workaround_env)
  90. .decode('utf8')
  91. .strip()
  92. .split('\n')
  93. )
  94. except Exception as e: # pylint: disable=W0703
  95. logging.warning(str(e))
  96. files = []
  97. return [os.path.join(path, f) for f in files] if full_path else files
  98. def is_in_directory(file_path: str, folder: str) -> bool:
  99. return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep)
  100. class PytestCase:
  101. def __init__(self, test_path: str, target: str, config: str, case: str):
  102. self.app_path = os.path.dirname(test_path)
  103. self.test_path = test_path
  104. self.target = target
  105. self.config = config
  106. self.case = case
  107. def __repr__(self) -> str:
  108. return f'{self.test_path}: {self.target}.{self.config}.{self.case}'
  109. class PytestCollectPlugin:
  110. def __init__(self, target: str) -> None:
  111. self.target = target
  112. self.nodes: List[PytestCase] = []
  113. def pytest_collection_modifyitems(self, items: List['Function']) -> None:
  114. for item in items:
  115. try:
  116. file_path = str(item.path)
  117. except AttributeError:
  118. # pytest 6.x
  119. file_path = item.fspath
  120. target = self.target
  121. if hasattr(item, 'callspec'):
  122. config = item.callspec.params.get('config', 'default')
  123. else:
  124. config = 'default'
  125. case_name = item.originalname
  126. self.nodes.append(PytestCase(file_path, target, config, case_name))
  127. def get_pytest_cases(folder: str, target: str) -> List[PytestCase]:
  128. import pytest
  129. from _pytest.config import ExitCode
  130. collector = PytestCollectPlugin(target)
  131. with io.StringIO() as buf:
  132. with redirect_stdout(buf):
  133. res = pytest.main(['--collect-only', folder, '-q', '--target', target], plugins=[collector])
  134. if res.value != ExitCode.OK:
  135. if res.value == ExitCode.NO_TESTS_COLLECTED:
  136. print(f'WARNING: no pytest app found for target {target} under folder {folder}')
  137. else:
  138. print(buf.getvalue())
  139. raise RuntimeError('pytest collection failed')
  140. return collector.nodes
  141. def get_pytest_app_paths(folder: str, target: str) -> List[str]:
  142. nodes = get_pytest_cases(folder, target)
  143. return list({node.app_path for node in nodes})