Merge branch 'feat/python_reqs_directory' into 'master'

Tools: Make easier the detection of the list of Python features

See merge request espressif/esp-idf!17009

.gitlab/CODEOWNERS

@@ -201,8 +201,6 @@
 
 /tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
 
-requirements.*.txt              @esp-idf-codeowners/tools
-
 # sort-order-reset
 
 /components/**/test_apps/**/*.py     @esp-idf-codeowners/ci @esp-idf-codeowners/tools

.gitlab/ci/rules.yml

@@ -47,6 +47,7 @@
   - "tools/cmake/**/*"
   - "tools/kconfig_new/**/*"
   - "tools/tools.json"
+  - "tools/requirements.json"
   - "tools/ci/test_build_system*.sh"
 
 .patterns-custom_test: &patterns-custom_test
@@ -127,6 +128,10 @@
   - "tools/test_idf_tools/**/*"
   - "tools/install_util.py"
 
+  - "tools/requirements/*"
+  - "tools/requirements.json"
+  - "tools/requirements_schema.json"
+
   - "tools/mkdfu.py"
   - "tools/test_mkdfu/**/*"
 

.pre-commit-config.yaml

@@ -111,6 +111,14 @@ repos:
         language: python
         files: \.(py|c|h|cpp|hpp|ld|s|S)$
         require_serial: true
+      - id: check-requirement-files
+        name: Check requirement files
+        entry: tools/ci/check_requirement_files.py
+        additional_dependencies:
+          - 'jsonschema'
+        language: python
+        files: 'tools/requirements.+|tools/requirements/.+'
+        pass_filenames: false
       - id: check-tools-files-patterns
         name: Check tools dir files patterns
         entry: tools/ci/check_tools_files_patterns.py

docs/en/api-guides/tools/idf-tools.rst

@@ -104,9 +104,9 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo
 
 * ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``.
 
-* ``install-python-env``: Create a Python virtual environment in the ``${IDF_TOOLS_PATH}/python_env`` directory and install there the required Python packages. An optional ``--features`` argument allows one to specify a comma-separated list of features. For each feature a requirements file must exist. For example, feature ``XY`` is a valid feature if ``${IDF_PATH}/requirements.XY.txt`` is an existing file with a list of Python packages to be installed. There is one mandatory ``core`` feature ensuring core functionality of ESP-IDF (build, flash, monitor, debug in console). There can be an arbitrary number of optional features. The selected list of features is stored in ``idf-env.json``. The requirement files contain a list of the desired Python packages to be installed and ``espidf.constraints.*.txt`` downloaded from https://dl.espressif.com and stored in ``${IDF_TOOLS_PATH}`` the package version requirements for a given ESP-IDF version.
+* ``install-python-env``: Create a Python virtual environment in the ``${IDF_TOOLS_PATH}/python_env`` directory and install there the required Python packages. An optional ``--features`` argument allows one to specify a comma-separated list of features. For each feature a requirements file must exist. For example, feature ``XY`` is a valid feature if ``${IDF_PATH}/tools/requirements/requirements.XY.txt`` is an existing file with a list of Python packages to be installed. There is one mandatory ``core`` feature ensuring core functionality of ESP-IDF (build, flash, monitor, debug in console). There can be an arbitrary number of optional features. The selected list of features is stored in ``idf-env.json``. The requirement files contain a list of the desired Python packages to be installed and ``espidf.constraints.*.txt`` downloaded from https://dl.espressif.com and stored in ``${IDF_TOOLS_PATH}`` the package version requirements for a given ESP-IDF version.
 
-* ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day.
+* ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/tools/requirements/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day.
 
 .. _idf-tools-install:
 

tools/ci/check_requirement_files.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+import re
+from typing import Any
+
+import jsonschema
+
+IDF_PATH = os.environ['IDF_PATH']
+JSON_PATH = os.path.join(IDF_PATH, 'tools', 'requirements.json')
+SCHEMA_PATH = os.path.join(IDF_PATH, 'tools', 'requirements_schema.json')
+REQ_DIR = os.path.join(IDF_PATH, 'tools', 'requirements')
+RE_FEATURE = re.compile(r'requirements\.(\w+)\.txt')
+
+
+def action_validate(req_obj: Any) -> None:  # "Any" because we are checking this in this script
+    '''
+    Check that the parsed JSON object is valid according to the JSON schema provided
+    '''
+    with open(SCHEMA_PATH, 'r') as schema_file:
+        schema_json = json.load(schema_file)
+        jsonschema.validate(req_obj, schema_json)
+
+
+def action_check_directory(req_obj: Any) -> None:  # "Any" because we are checking this in this script
+    '''
+    Check that all directory items are listed in the JSON file
+    '''
+    features = set(d['name'] for d in req_obj['features'])
+    features_found = set()
+
+    for file_name in os.listdir(REQ_DIR):
+        m = re.match(RE_FEATURE, file_name)
+        if m:
+            if m.group(1) not in features:
+                raise RuntimeError(f'Cannot find a feature for {file_name} in {JSON_PATH}')
+            features_found.add(m.group(1))
+        else:
+            raise RuntimeError(f'{file_name} in {REQ_DIR} doesn\'t match the expected name')
+
+    features_not_found = features - features_found
+
+    if len(features_not_found) > 0:
+        raise RuntimeError(f'There are no requirements file in {REQ_DIR} for {features_not_found}')
+
+
+def main() -> None:
+    with open(JSON_PATH, 'r') as f:
+        req_obj = json.load(f)
+    action_validate(req_obj)
+    action_check_directory(req_obj)
+
+
+if __name__ == '__main__':
+    main()

tools/ci/executable-list.txt

@@ -60,6 +60,7 @@ tools/ci/check_executables.py
 tools/ci/check_idf_version.sh
 tools/ci/check_kconfigs.py
 tools/ci/check_readme_links.py
+tools/ci/check_requirement_files.py
 tools/ci/check_rules_yml.py
 tools/ci/check_soc_struct_headers.py
 tools/ci/check_tools_files_patterns.py

tools/idf_tools.py

@@ -1092,7 +1092,7 @@ def add_and_save_targets(targets_str):  # type: (str) -> list[str]
 
 
 def feature_to_requirements_path(feature):  # type: (str) -> str
-    return os.path.join(global_idf_path or '', 'requirements.{}.txt'.format(feature))
+    return os.path.join(global_idf_path or '', 'tools', 'requirements', 'requirements.{}.txt'.format(feature))
 
 
 def add_and_save_features(features_str):  # type: (str) -> list[str]

tools/requirements.json

@@ -0,0 +1,17 @@
+{
+    "version": 1,
+    "features": [
+        {
+            "name": "core",
+            "description": "Core packages necessary for ESP-IDF",
+            "optional": false,
+            "requirement_path": "tools/requirements/requirements.core.txt"
+        },
+        {
+            "name": "gdbgui",
+            "description": "Packages for supporting debugging from web browser",
+            "optional": true,
+            "requirement_path": "tools/requirements/requirements.gdbgui.txt"
+        }
+    ]
+}

tools/requirements_schema.json

@@ -0,0 +1,52 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "https://github.com/espressif/esp-idf/blob/master/tools/requirements_schema.json",
+    "type": "object",
+    "properties": {
+        "version": {
+            "type": "integer",
+            "description": "Metadata file version"
+        },
+        "features": {
+            "type": "array",
+            "description": "List of features",
+            "items": {
+                "$ref": "#/definitions/featInfo"
+            }
+        }
+    },
+    "required": [
+        "version",
+        "features"
+    ],
+    "definitions": {
+        "featInfo": {
+            "type": "object",
+            "description": "Information about one feature",
+            "properties": {
+                "name" : {
+                    "description": "Feature name",
+                    "type": "string"
+                },
+                "description" : {
+                    "description": "A short description of the feature",
+                    "type": "string"
+                },
+                "optional": {
+                    "description": "The feature is optional if the user can choose to not install it",
+                    "type": "boolean"
+                },
+                "requirement_path": {
+                    "description": "Path to the requirements file with Python packages",
+                    "type": "string",
+                    "pattern": "^tools/requirements/requirements\\..+\\.txt$"
+                }
+            },
+            "required": [
+                "name",
+                "optional",
+                "requirement_path"
+            ]
+        }
+    }
+}

tools/test_idf_tools/test_idf_tools_python_env.py

@@ -18,8 +18,8 @@ IDF_PATH = os.environ.get('IDF_PATH', '../..')
 TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
 PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env')
 REQ_SATISFIED = 'Python requirements are satisfied'
-REQ_CORE = '- {}/requirements.core.txt'.format(IDF_PATH)
-REQ_GDBGUI = '- {}/requirements.gdbgui.txt'.format(IDF_PATH)
+REQ_CORE = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.core.txt'))
+REQ_GDBGUI = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.gdbgui.txt'))
 CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR)