Merge branch 'feature/idfpy_unknown_targets_fallback' into 'master'

idf.py: run build system target for unknown sub-commands

Closes IDF-748

See merge request espressif/esp-idf!6644

docs/en/api-guides/build-system.rst

@@ -67,7 +67,7 @@ The :ref:`getting started guide <get-started-configure>` contains a brief introd
 
 ``idf.py`` should be run in an ESP-IDF "project" directory, ie one containing a ``CMakeLists.txt`` file. Older style projects with a Makefile will not work with ``idf.py``.
 
-Type ``idf.py --help`` for a full list of commands. Here are a summary of the most useful ones:
+Type ``idf.py --help`` for a list of commands. Here are a summary of the most useful ones:
 
 - ``idf.py menuconfig`` runs the "menuconfig" tool to configure the project.
 - ``idf.py build`` will build the project found in the current directory. This can involve multiple steps:
@@ -84,6 +84,8 @@ Type ``idf.py --help`` for a full list of commands. Here are a summary of the mo
 
 Multiple ``idf.py`` commands can be combined into one. For example, ``idf.py -p COM4 clean flash monitor`` will clean the source tree, then build the project and flash it to the ESP32 before running the serial monitor.
 
+For commands that are not known to ``idf.py`` an attempt to execute them as a build system target will be made.
+
 .. note:: The environment variables ``ESPPORT`` and ``ESPBAUD`` can be used to set default values for the ``-p`` and ``-b`` options, respectively. Providing these options on the command line overrides the default.
 
 .. _idf.py-size:
@@ -101,8 +103,8 @@ The order of multiple ``idf.py`` commands on the same invocation is not importan
 
 idf.py options
 ^^^^^^^^^^^^^^
-
-To list all available options, run ``idf.py --help``.
+To list all available root level options, run ``idf.py --help``. To list options that are specific for a subcommand, run ``idf.py <command> --help``, for example ``idf.py monitor --help``.
+Here is a list of some useful options:
 
 - ``-C <dir>`` allows overriding the project directory from the default current working directory.
 - ``-B <dir>`` allows overriding the build directory from the default ``build`` subdirectory of the project directory.

tools/idf.py

@@ -70,9 +70,10 @@ def check_environment():
     if "IDF_PATH" in os.environ:
         set_idf_path = realpath(os.environ["IDF_PATH"])
         if set_idf_path != detected_idf_path:
-            print("WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
-                  "Using the environment variable directory, but results may be unexpected..." %
-                  (set_idf_path, PROG, detected_idf_path))
+            print(
+                "WARNING: IDF_PATH environment variable is set to %s but %s path indicates IDF directory %s. "
+                "Using the environment variable directory, but results may be unexpected..." %
+                (set_idf_path, PROG, detected_idf_path))
     else:
         print("Setting IDF_PATH environment variable: %s" % detected_idf_path)
         os.environ["IDF_PATH"] = detected_idf_path
@@ -189,14 +190,15 @@ def init_cli(verbose_output=None):
             self.callback(self.name, context, global_args, **action_args)
 
     class Action(click.Command):
-        def __init__(self,
-                     name=None,
-                     aliases=None,
-                     deprecated=False,
-                     dependencies=None,
-                     order_dependencies=None,
-                     hidden=False,
-                     **kwargs):
+        def __init__(
+                self,
+                name=None,
+                aliases=None,
+                deprecated=False,
+                dependencies=None,
+                order_dependencies=None,
+                hidden=False,
+                **kwargs):
             super(Action, self).__init__(name, **kwargs)
 
             self.name = self.name or self.callback.__name__
@@ -232,12 +234,12 @@ def init_cli(verbose_output=None):
                 self.help = "\n".join([self.help, aliases_help])
                 self.short_help = " ".join([aliases_help, self.short_help])
 
+            self.unwrapped_callback = self.callback
             if self.callback is not None:
-                callback = self.callback
 
                 def wrapped_callback(**action_args):
                     return Task(
-                        callback=callback,
+                        callback=self.unwrapped_callback,
                         name=self.name,
                         dependencies=dependencies,
                         order_dependencies=order_dependencies,
@@ -397,8 +399,9 @@ def init_cli(verbose_output=None):
                     option = Option(**option_args)
 
                     if option.scope.is_shared:
-                        raise FatalError('"%s" is defined for action "%s". '
-                                         ' "shared" options can be declared only on global level' % (option.name, name))
+                        raise FatalError(
+                            '"%s" is defined for action "%s". '
+                            ' "shared" options can be declared only on global level' % (option.name, name))
 
                     # Promote options to global if see for the first time
                     if option.scope.is_global and option.name not in [o.name for o in self.params]:
@@ -410,7 +413,12 @@ def init_cli(verbose_output=None):
             return sorted(filter(lambda name: not self._actions[name].hidden, self._actions))
 
         def get_command(self, ctx, name):
-            return self._actions.get(self.commands_with_aliases.get(name))
+            if name in self.commands_with_aliases:
+                return self._actions.get(self.commands_with_aliases.get(name))
+
+            # Trying fallback to build target (from "all" action) if command is not known
+            else:
+                return Action(name=name, callback=self._actions.get('fallback').unwrapped_callback)
 
         def _print_closing_message(self, args, actions):
             # print a closing message of some kind
@@ -450,19 +458,21 @@ def init_cli(verbose_output=None):
                     for o, f in flash_items:
                         cmd += o + " " + flasher_path(f) + " "
 
-                print("%s %s -p %s -b %s --before %s --after %s write_flash %s" % (
-                    PYTHON,
-                    _safe_relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]),
-                    args.port or "(PORT)",
-                    args.baud,
-                    flasher_args["extra_esptool_args"]["before"],
-                    flasher_args["extra_esptool_args"]["after"],
-                    cmd.strip(),
-                ))
-                print("or run 'idf.py -p %s %s'" % (
-                    args.port or "(PORT)",
-                    key + "-flash" if key != "project" else "flash",
-                ))
+                print(
+                    "%s %s -p %s -b %s --before %s --after %s write_flash %s" % (
+                        PYTHON,
+                        _safe_relpath("%s/components/esptool_py/esptool/esptool.py" % os.environ["IDF_PATH"]),
+                        args.port or "(PORT)",
+                        args.baud,
+                        flasher_args["extra_esptool_args"]["before"],
+                        flasher_args["extra_esptool_args"]["after"],
+                        cmd.strip(),
+                    ))
+                print(
+                    "or run 'idf.py -p %s %s'" % (
+                        args.port or "(PORT)",
+                        key + "-flash" if key != "project" else "flash",
+                    ))
 
             if "all" in actions or "build" in actions:
                 print_flashing_message("Project", "project")
@@ -483,9 +493,10 @@ def init_cli(verbose_output=None):
                 [item for item, count in Counter(task.name for task in tasks).items() if count > 1])
             if dupplicated_tasks:
                 dupes = ", ".join('"%s"' % t for t in dupplicated_tasks)
-                print("WARNING: Command%s found in the list of commands more than once. " %
-                      ("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes) +
-                      "Only first occurence will be executed.")
+                print(
+                    "WARNING: Command%s found in the list of commands more than once. " %
+                    ("s %s are" % dupes if len(dupplicated_tasks) > 1 else " %s is" % dupes) +
+                    "Only first occurence will be executed.")
 
             # Set propagated global options.
             # These options may be set on one subcommand, but available in the list of global arguments
@@ -499,9 +510,9 @@ def init_cli(verbose_output=None):
                         default = () if option.multiple else option.default
 
                         if global_value != default and local_value != default and global_value != local_value:
-                            raise FatalError('Option "%s" provided for "%s" is already defined to a different value. '
-                                             "This option can appear at most once in the command line." %
-                                             (key, task.name))
+                            raise FatalError(
+                                'Option "%s" provided for "%s" is already defined to a different value. '
+                                "This option can appear at most once in the command line." % (key, task.name))
                         if local_value != default:
                             global_args[key] = local_value
 
@@ -537,8 +548,9 @@ def init_cli(verbose_output=None):
                         # Otherwise invoke it with default set of options
                         # and put to the front of the list of unprocessed tasks
                         else:
-                            print('Adding "%s"\'s dependency "%s" to list of commands with default set of options.' %
-                                  (task.name, dep))
+                            print(
+                                'Adding "%s"\'s dependency "%s" to list of commands with default set of options.' %
+                                (task.name, dep))
                             dep_task = ctx.invoke(ctx.command.get_command(ctx, dep))
 
                             # Remove options with global scope from invoke tasks because they are alread in global_args
@@ -631,7 +643,11 @@ def init_cli(verbose_output=None):
         except NameError:
             pass
 
-    return CLI(help="ESP-IDF build management", verbose_output=verbose_output, all_actions=all_actions)
+    cli_help = (
+        "ESP-IDF CLI build management tool. "
+        "For commands that are not known to idf.py an attempt to execute it as a build system target will be made.")
+
+    return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions)
 
 
 def main():
@@ -709,8 +725,9 @@ if __name__ == "__main__":
             # Trying to find best utf-8 locale available on the system and restart python with it
             best_locale = _find_usable_locale()
 
-            print("Your environment is not configured to handle unicode filenames outside of ASCII range."
-                  " Environment variable LC_ALL is temporary set to %s for unicode support." % best_locale)
+            print(
+                "Your environment is not configured to handle unicode filenames outside of ASCII range."
+                " Environment variable LC_ALL is temporary set to %s for unicode support." % best_locale)
 
             os.environ["LC_ALL"] = best_locale
             ret = subprocess.call([sys.executable] + sys.argv, env=os.environ)

tools/idf_py_actions/constants.py

@@ -16,17 +16,23 @@ else:
     MAKE_CMD = "make"
     MAKE_GENERATOR = "Unix Makefiles"
 
-GENERATORS = [
-    # ('generator name', 'build command line', 'version command line', 'verbose flag')
-    ("Ninja", ["ninja"], ["ninja", "--version"], "-v"),
-    (
-        MAKE_GENERATOR,
-        [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)],
-        [MAKE_CMD, "--version"],
-        "VERBOSE=1",
-    ),
-]
-GENERATOR_CMDS = dict((a[0], a[1]) for a in GENERATORS)
-GENERATOR_VERBOSE = dict((a[0], a[3]) for a in GENERATORS)
+GENERATORS = {
+    # - command: build command line
+    # - version: version command line
+    # - dry_run: command to run in dry run mode
+    # - verbose_flag: verbose flag
+    "Ninja": {
+        "command": ["ninja"],
+        "version": ["ninja", "--version"],
+        "dry_run": ["ninja", "-n"],
+        "verbose_flag": "-v"
+    },
+    MAKE_GENERATOR: {
+        "command": [MAKE_CMD, "-j", str(multiprocessing.cpu_count() + 2)],
+        "version": [MAKE_CMD, "--version"],
+        "dry_run": [MAKE_CMD, "-n"],
+        "verbose_flag": "VERBOSE=1",
+    }
+}
 
 SUPPORTED_TARGETS = ["esp32", "esp32s2beta"]

tools/idf_py_actions/core_ext.py

@@ -1,16 +1,25 @@
 import os
 import shutil
+import subprocess
 import sys
 
 import click
 
-from idf_py_actions.constants import GENERATOR_CMDS, GENERATOR_VERBOSE, SUPPORTED_TARGETS
+from idf_py_actions.constants import GENERATORS, SUPPORTED_TARGETS
 from idf_py_actions.errors import FatalError
 from idf_py_actions.global_options import global_options
 from idf_py_actions.tools import ensure_build_directory, idf_version, merge_action_lists, realpath, run_tool
 
 
 def action_extensions(base_actions, project_path):
+    def run_target(target_name, args):
+        generator_cmd = GENERATORS[args.generator]["command"]
+
+        if args.verbose:
+            generator_cmd += [GENERATORS[args.generator]["verbose_flag"]]
+
+        run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir)
+
     def build_target(target_name, ctx, args):
         """
         Execute the target build system to build target 'target_name'
@@ -19,12 +28,23 @@ def action_extensions(base_actions, project_path):
         directory (with the specified generator) as needed.
         """
         ensure_build_directory(args, ctx.info_name)
-        generator_cmd = GENERATOR_CMDS[args.generator]
+        run_target(target_name, args)
 
-        if args.verbose:
-            generator_cmd += [GENERATOR_VERBOSE[args.generator]]
+    def fallback_target(target_name, ctx, args):
+        """
+        Execute targets that are not explicitly known to idf.py
+        """
+        ensure_build_directory(args, ctx.info_name)
 
-        run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir)
+        try:
+            subprocess.check_output(GENERATORS[args.generator]["dry_run"] + [target_name], cwd=args.cwd)
+
+        except Exception:
+            raise FatalError(
+                'command "%s" is not known to idf.py and is not a %s target' %
+                (target_name, GENERATORS[args.generator].command))
+
+        run_target(target_name, args)
 
     def verbose_callback(ctx, param, value):
         if not value or ctx.resilient_parsing:
@@ -69,8 +89,9 @@ def action_extensions(base_actions, project_path):
             return
 
         if not os.path.exists(os.path.join(build_dir, "CMakeCache.txt")):
-            raise FatalError("Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically "
-                             "delete files in this directory. Delete the directory manually to 'clean' it." % build_dir)
+            raise FatalError(
+                "Directory '%s' doesn't seem to be a CMake build directory. Refusing to automatically "
+                "delete files in this directory. Delete the directory manually to 'clean' it." % build_dir)
         red_flags = ["CMakeLists.txt", ".git", ".svn"]
         for red in red_flags:
             red = os.path.join(build_dir, red)
@@ -111,8 +132,9 @@ def action_extensions(base_actions, project_path):
     def validate_root_options(ctx, args, tasks):
         args.project_dir = realpath(args.project_dir)
         if args.build_dir is not None and args.project_dir == realpath(args.build_dir):
-            raise FatalError("Setting the build directory to the project directory is not supported. Suggest dropping "
-                             "--build-dir option, the default is a 'build' subdirectory inside the project directory.")
+            raise FatalError(
+                "Setting the build directory to the project directory is not supported. Suggest dropping "
+                "--build-dir option, the default is a 'build' subdirectory inside the project directory.")
         if args.build_dir is None:
             args.build_dir = os.path.join(args.project_dir, "build")
         args.build_dir = realpath(args.build_dir)
@@ -165,15 +187,16 @@ def action_extensions(base_actions, project_path):
             },
             {
                 "names": ["--ccache/--no-ccache"],
-                "help": ("Use ccache in build. Disabled by default, unless "
-                         "IDF_CCACHE_ENABLE environment variable is set to a non-zero value."),
+                "help": (
+                    "Use ccache in build. Disabled by default, unless "
+                    "IDF_CCACHE_ENABLE environment variable is set to a non-zero value."),
                 "is_flag": True,
                 "default": os.getenv("IDF_CCACHE_ENABLE") not in [None, "", "0"],
             },
             {
                 "names": ["-G", "--generator"],
                 "help": "CMake generator.",
-                "type": click.Choice(GENERATOR_CMDS.keys()),
+                "type": click.Choice(GENERATORS.keys()),
             },
             {
                 "names": ["--dry-run"],
@@ -192,15 +215,16 @@ def action_extensions(base_actions, project_path):
                 "aliases": ["build"],
                 "callback": build_target,
                 "short_help": "Build the project.",
-                "help": ("Build the project. This can involve multiple steps:\n\n"
-                         "1. Create the build directory if needed. "
-                         "The sub-directory 'build' is used to hold build output, "
-                         "although this can be changed with the -B option.\n\n"
-                         "2. Run CMake as necessary to configure the project "
-                         "and generate build files for the main build tool.\n\n"
-                         "3. Run the main build tool (Ninja or GNU Make). "
-                         "By default, the build tool is automatically detected "
-                         "but it can be explicitly set by passing the -G option to idf.py.\n\n"),
+                "help": (
+                    "Build the project. This can involve multiple steps:\n\n"
+                    "1. Create the build directory if needed. "
+                    "The sub-directory 'build' is used to hold build output, "
+                    "although this can be changed with the -B option.\n\n"
+                    "2. Run CMake as necessary to configure the project "
+                    "and generate build files for the main build tool.\n\n"
+                    "3. Run the main build tool (Ninja or GNU Make). "
+                    "By default, the build tool is automatically detected "
+                    "but it can be explicitly set by passing the -G option to idf.py.\n\n"),
                 "options": global_options,
                 "order_dependencies": [
                     "reconfigure",
@@ -282,6 +306,11 @@ def action_extensions(base_actions, project_path):
                 "help": "Read otadata partition.",
                 "options": global_options,
             },
+            "fallback": {
+                "callback": fallback_target,
+                "help": "Handle for targets not known for idf.py.",
+                "hidden": True
+            }
         }
     }
 
@@ -290,23 +319,25 @@ def action_extensions(base_actions, project_path):
             "reconfigure": {
                 "callback": reconfigure,
                 "short_help": "Re-run CMake.",
-                "help": ("Re-run CMake even if it doesn't seem to need re-running. "
-                         "This isn't necessary during normal usage, "
-                         "but can be useful after adding/removing files from the source tree, "
-                         "or when modifying CMake cache variables. "
-                         "For example, \"idf.py -DNAME='VALUE' reconfigure\" "
-                         'can be used to set variable "NAME" in CMake cache to value "VALUE".'),
+                "help": (
+                    "Re-run CMake even if it doesn't seem to need re-running. "
+                    "This isn't necessary during normal usage, "
+                    "but can be useful after adding/removing files from the source tree, "
+                    "or when modifying CMake cache variables. "
+                    "For example, \"idf.py -DNAME='VALUE' reconfigure\" "
+                    'can be used to set variable "NAME" in CMake cache to value "VALUE".'),
                 "options": global_options,
                 "order_dependencies": ["menuconfig", "fullclean"],
             },
             "set-target": {
                 "callback": set_target,
                 "short_help": "Set the chip target to build.",
-                "help": ("Set the chip target to build. This will remove the "
-                         "existing sdkconfig file and corresponding CMakeCache and "
-                         "create new ones according to the new target.\nFor example, "
-                         "\"idf.py set-target esp32\" will select esp32 as the new chip "
-                         "target."),
+                "help": (
+                    "Set the chip target to build. This will remove the "
+                    "existing sdkconfig file and corresponding CMakeCache and "
+                    "create new ones according to the new target.\nFor example, "
+                    "\"idf.py set-target esp32\" will select esp32 as the new chip "
+                    "target."),
                 "arguments": [
                     {
                         "names": ["idf-target"],
@@ -319,22 +350,24 @@ def action_extensions(base_actions, project_path):
             "clean": {
                 "callback": clean,
                 "short_help": "Delete build output files from the build directory.",
-                "help": ("Delete build output files from the build directory, "
-                         "forcing a 'full rebuild' the next time "
-                         "the project is built. Cleaning doesn't delete "
-                         "CMake configuration output and some other files"),
+                "help": (
+                    "Delete build output files from the build directory, "
+                    "forcing a 'full rebuild' the next time "
+                    "the project is built. Cleaning doesn't delete "
+                    "CMake configuration output and some other files"),
                 "order_dependencies": ["fullclean"],
             },
             "fullclean": {
                 "callback": fullclean,
                 "short_help": "Delete the entire build directory contents.",
-                "help": ("Delete the entire build directory contents. "
-                         "This includes all CMake configuration output."
-                         "The next time the project is built, "
-                         "CMake will configure it from scratch. "
-                         "Note that this option recursively deletes all files "
-                         "in the build directory, so use with care."
-                         "Project configuration is not deleted.")
+                "help": (
+                    "Delete the entire build directory contents. "
+                    "This includes all CMake configuration output."
+                    "The next time the project is built, "
+                    "CMake will configure it from scratch. "
+                    "Note that this option recursively deletes all files "
+                    "in the build directory, so use with care."
+                    "Project configuration is not deleted.")
             },
         }
     }

tools/idf_py_actions/tools.py

@@ -126,9 +126,9 @@ def _detect_cmake_generator(prog_name):
     """
     Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found.
     """
-    for (generator, _, version_check, _) in GENERATORS:
-        if executable_exists(version_check):
-            return generator
+    for (generator_name,  generator) in GENERATORS.items():
+        if executable_exists(generator["version"]):
+            return generator_name
     raise FatalError("To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" % prog_name)