Merge branch 'feature/easy_debug' into 'master'

idf.py: add debug related targets

Closes IDF-890

See merge request espressif/esp-idf!5602

docs/en/api-guides/jtag-debugging/using-debugger.rst

@@ -2,8 +2,9 @@ Using Debugger
 --------------
 :link_to_translation:`zh_CN:[中文]`
 
-This section covers configuration and running debugger either from :ref:`jtag-debugging-using-debugger-eclipse`
-or :ref:`jtag-debugging-using-debugger-command-line`. It is recommended to first check if debugger works from :ref:`jtag-debugging-using-debugger-command-line` and then move to using Eclipse.
+This section covers configuration and running debugger from :ref:`jtag-debugging-using-debugger-eclipse`
+or from :ref:`jtag-debugging-using-debugger-command-line` or using :ref:`jtag-debugging-with-idf-py`.
+It is recommended to first check if debugger works from :ref:`jtag-debugging-using-debugger-command-line` and then move to using Eclipse.
 
 
 .. _jtag-debugging-using-debugger-eclipse:
@@ -188,3 +189,54 @@ Command Line
 Note the third line from bottom that shows debugger halting at breakpoint established in ``gdbinit`` file at function ``app_main()``. Since the processor is halted, the LED should not be blinking. If this is what you see as well, you are ready to start debugging.
 
 If you are not quite sure how to use GDB, check :ref:`jtag-debugging-examples-command-line` example debugging session in section :ref:`jtag-debugging-examples`.
+
+
+.. _jtag-debugging-with-idf-py:
+
+Using idf.py debug targets
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is also possible to execute the described debugging tools conveniently from ``idf.py``. These commands are supported:
+
+1. ``idf.py openocd``
+
+Runs OpenOCD in a console with configuration defined in the environment or via command line.
+It uses default script directory defined as ``OPENOCD_SCRIPTS`` environmental variable, which is automatically added
+from an Export script (``export.sh`` or ``export.bat``). It is possible to override the script location
+using command line argument ``--openocd-scripts``.
+
+As for the JTAG configuration of the current board, please use the environmental variable ``OPENOCD_COMMANDS``
+or ``--openocd-commands`` command line argument. If none of the above is defined,
+OpenOCD is started with ``-f board/esp32-wrover-kit-3.3v.cfg`` board definition.
+
+
+2. ``idf.py gdb``
+
+Starts the gdb the same way as the :ref:`jtag-debugging-using-debugger-command-line`, but generates the initial gdb scripts
+referring to the current project elf file.
+
+
+3. ``idf.py gdbtui``
+
+The same as `2`, but starts the gdb with ``tui`` argument allowing very simple source code view.
+
+
+4. ``idf.py gdbgui``
+
+Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window.
+
+
+It is possible to combine these debugging actions on a single command line allowing convenient 
+setup of blocking and non-blocking actions in one step. ``idf.py`` implements a simple logic to move the background
+actions (such as openocd) to the beginning and the interactive ones (such as gdb, monitor) to the end of the action list.
+
+An example of a very useful combination is
+
+    ::
+
+        idf.py openocd gdbgui monitor
+
+.. highlight:: none
+
+The above command runs OpenOCD in the background, starts `gdbgui <https://www.gdbgui.com>`_ to open a browser window
+with active debugger frontend and opens a serial monitor in the active console.

requirements.txt

@@ -11,7 +11,10 @@ future>=0.15.2
 cryptography>=2.1.4
 pyparsing>=2.0.3,<2.4.0
 pyelftools>=0.22
- 
+gdbgui>=0.13.2.0
+pygdbmi<=0.9.0.2
+# The pygdbmi required max version 0.9.0.2 since 0.9.0.3 is not copatible with latest gdbgui (>=0.13.2.0)
+
 # windows-curses are required in Windows command line but cannot be installed in MSYS2. A requirement like
 # "windows-curses; sys_platform == 'win32'" would want to install the package on both of them. There is no environment
 # marker for detecting MSYS2. So instead, a dummy custom package is used with "windows-curses" dependency for Windows

tools/idf_py_actions/debug_ext.py

@@ -0,0 +1,351 @@
+import json
+import os
+import sys
+import subprocess
+import shlex
+import time
+import re
+from threading import Thread
+import threading
+
+from idf_py_actions.errors import FatalError
+from idf_py_actions.tools import ensure_build_directory
+
+PYTHON = sys.executable
+
+
+def action_extensions(base_actions, project_path):
+    OPENOCD_OUT_FILE = "openocd_out.txt"
+    GDBGUI_OUT_FILE = "gdbgui_out.txt"
+    # Internal dictionary of currently active processes, threads and their output files
+    processes = {"threads_to_join": []}
+
+    def _check_openocd_errors(fail_if_openocd_failed, target, ctx):
+        if fail_if_openocd_failed:
+            if "openocd" in processes and processes["openocd"] is not None:
+                p = processes["openocd"]
+                name = processes["openocd_outfile_name"]
+                # watch OpenOCD (for 5x500ms) to check if it hasn't terminated or outputs an error
+                for _ in range(5):
+                    if p.poll() is not None:
+                        print("OpenOCD exited with {}".format(p.poll()))
+                        break
+                    with open(name, "r") as f:
+                        content = f.read()
+                        if re.search(r"no device found", content):
+                            break
+                        if re.search(r"Listening on port \d+ for gdb connections", content):
+                            # expect OpenOCD has started successfully - stop watching
+                            return
+                    time.sleep(0.5)
+                else:
+                    return
+                # OpenOCD exited or error message detected -> print possible output and terminate
+                with open(name, "r") as f:
+                    print(f.read())
+                raise FatalError('Action "{}" failed due to errors in OpenOCD: Please check jtag connection!'.format(target), ctx)
+
+    def _terminate_async_target(target):
+        if target in processes and processes[target] is not None:
+            try:
+                if target + "_outfile" in processes:
+                    processes[target + "_outfile"].close()
+                p = processes[target]
+                if p.poll() is None:
+                    p.terminate()
+                    # waiting 10x100ms for the process to terminate gracefully
+                    for _ in range(10):
+                        if p.poll() is not None:
+                            break
+                        time.sleep(0.1)
+                    else:
+                        p.kill()
+                if target + "_outfile_name" in processes:
+                    os.unlink(processes[target + "_outfile_name"])
+            except Exception as e:
+                print(e)
+                print("Failed to close/kill {}".format(target))
+            processes[target] = None  # to indicate this has ended
+
+    def _get_commandline_options(ctx):
+        """ Return all the command line options up to first action """
+        # This approach ignores argument parsing done Click
+        result = []
+
+        for arg in sys.argv:
+            if arg in ctx.command.commands_with_aliases:
+                break
+
+            result.append(arg)
+
+        return result
+
+    def create_local_gdbinit(gdbinit, elf_file):
+        with open(gdbinit, "w") as f:
+            f.write("target remote :3333\n")
+            if os.name == "nt":
+                elf_file = elf_file.replace('\\','\\\\')
+            f.write("symbol-file {}\n".format(elf_file))
+            f.write("mon reset halt\n")
+            f.write("flushregs\n")
+            f.write("thb app_main\n")
+            f.write("c\n")
+
+    def debug_cleanup():
+        print("cleaning up debug targets")
+        for t in processes["threads_to_join"]:
+            if threading.currentThread() != t:
+                t.join()
+        _terminate_async_target("openocd")
+        _terminate_async_target("gdbgui")
+        _terminate_async_target("gdb")
+
+    def post_debug(action, ctx, args, block):
+        """ Deal with asynchronous targets, such as openocd running in background """
+        if block == 1:
+            for target in ["openocd", "gdbgui"]:
+                if target in processes and processes[target] is not None:
+                    break
+            else:
+                return
+            try:
+                p = processes[target]
+                name = processes[target + "_outfile_name"]
+                pos = 0
+                while True:
+                    with open(name, "r") as f:
+                        f.seek(pos)
+                        for line in f:
+                            print(line.rstrip())
+                        pos = f.tell()
+                    if p.poll() is not None:
+                        print('"{}" exited with {}'.format(target, p.poll()))
+                        break
+                    time.sleep(0.5)
+            except KeyboardInterrupt:
+                print("Terminated -> exiting debug utility targets")
+        _terminate_async_target("openocd")
+        _terminate_async_target("gdbgui")
+
+    def get_project_desc(args, ctx):
+        desc_path = os.path.join(args.build_dir, "project_description.json")
+        if not os.path.exists(desc_path):
+            ensure_build_directory(args, ctx.info_name)
+        with open(desc_path, "r") as f:
+            project_desc = json.load(f)
+            return project_desc
+
+    def openocd(action, ctx, args, openocd_scripts, openocd_commands):
+        """
+        Execute openocd as external tool
+        """
+        if os.getenv("OPENOCD_SCRIPTS") is None:
+            raise FatalError("OPENOCD_SCRIPTS not found in the environment: Please run export.sh/export.bin", ctx)
+        openocd_arguments = os.getenv("OPENOCD_COMMANDS") if openocd_commands is None else openocd_commands
+        project_desc = get_project_desc(args, ctx)
+        if openocd_arguments is None:
+            # use default value if commands not defined in the environment nor command line
+            if project_desc["target"] == "esp32":
+                openocd_arguments = "-f board/esp32-wrover-kit-3.3v.cfg"
+            else:
+                openocd_arguments = "-f interface/ftdi/esp32_devkitj_v1.cfg -f target/{}.cfg".format(project_desc["target"])
+            print('Note: OpenOCD cfg not found (via env variable OPENOCD_COMMANDS nor as a --openocd-commands argument)\n'
+                  'OpenOCD arguments default to: "{}"'.format(openocd_arguments))
+        # script directory is taken from the environment by OpenOCD, update only if command line arguments to override
+        if openocd_scripts is not None:
+            openocd_arguments += " -s {}".format(openocd_scripts)
+        local_dir = project_desc["build_dir"]
+        args = ["openocd"] + shlex.split(openocd_arguments)
+        openocd_out_name = os.path.join(local_dir, OPENOCD_OUT_FILE)
+        openocd_out = open(openocd_out_name, "a+")
+        try:
+            process = subprocess.Popen(args, stdout=openocd_out, stderr=subprocess.STDOUT, bufsize=1)
+        except Exception as e:
+            print(e)
+            raise FatalError("Error starting openocd. Please make sure it is installed and is present in executable paths", ctx)
+
+        processes["openocd"] = process
+        processes["openocd_outfile"] = openocd_out
+        processes["openocd_outfile_name"] = openocd_out_name
+        print("OpenOCD started as a background task {}".format(process.pid))
+
+    def gdbui(action, ctx, args, gdbgui_port, require_openocd):
+        """
+        Asynchronous GDB-UI target
+        """
+        project_desc = get_project_desc(args, ctx)
+        local_dir = project_desc["build_dir"]
+        gdb = project_desc["monitor_toolprefix"] + "gdb"
+        gdbinit = os.path.join(local_dir, 'gdbinit')
+        create_local_gdbinit(gdbinit, os.path.join(args.build_dir, project_desc["app_elf"]))
+        args = ["gdbgui", "-g", gdb, '--gdb-args="-x={}"'.format(gdbinit)]
+        if gdbgui_port is not None:
+            args += ["--port", gdbgui_port]
+        gdbgui_out_name = os.path.join(local_dir, GDBGUI_OUT_FILE)
+        gdbgui_out = open(gdbgui_out_name, "a+")
+        try:
+            process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1)
+        except Exception as e:
+            print(e)
+            raise FatalError("Error starting gdbgui. Please make sure gdbgui can be started", ctx)
+
+        processes["gdbgui"] = process
+        processes["gdbgui_outfile"] = gdbgui_out
+        processes["gdbgui_outfile_name"] = gdbgui_out_name
+        print("gdbgui started as a background task {}".format(process.pid))
+        _check_openocd_errors(fail_if_openocd_failed, action, ctx)
+
+    def global_callback(ctx, global_args, tasks):
+        def move_to_front(task_name):
+            for index, task in enumerate(tasks):
+                if task.name == task_name:
+                    tasks.insert(0, tasks.pop(index))
+                    break
+
+        debug_targets = any([task.name in ("openocd", "gdbgui") for task in tasks])
+        if debug_targets:
+            # Register the meta cleanup callback -> called on FatalError
+            ctx.meta["cleanup"] = debug_cleanup
+            move_to_front("gdbgui")     # possibly 2nd
+            move_to_front("openocd")    # always 1st
+            # followed by "monitor", "gdb" or "gdbtui" in any order
+
+            post_action = ctx.invoke(ctx.command.get_command(ctx, "post_debug"))
+            if any([task.name in ("monitor", "gdb", "gdbtui") for task in tasks]):
+                post_action.action_args["block"] = 0
+            else:
+                post_action.action_args["block"] = 1
+            tasks.append(post_action)   # always last
+        if any([task.name == "openocd" for task in tasks]):
+            for task in tasks:
+                if task.name in ("gdb", "gdbgui", "gdbtui"):
+                    task.action_args["require_openocd"] = True
+
+    def run_gdb(gdb_args):
+        p = subprocess.Popen(gdb_args)
+        processes["gdb"] = p
+        return p.wait()
+
+    def gdbtui(action, ctx, args, require_openocd):
+        """
+        Synchronous GDB target with text ui mode
+        """
+        gdb(action, ctx, args, 1, require_openocd)
+
+    def gdb(action, ctx, args, gdb_tui, require_openocd):
+        """
+        Synchronous GDB target
+        """
+        watch_openocd = Thread(target=_check_openocd_errors, args=(fail_if_openocd_failed, action, ctx, ))
+        watch_openocd.start()
+        processes["threads_to_join"].append(watch_openocd)
+        desc_path = os.path.join(args.build_dir, "project_description.json")
+        if not os.path.exists(desc_path):
+            ensure_build_directory(args, ctx.info_name)
+        with open(desc_path, "r") as f:
+            project_desc = json.load(f)
+
+        elf_file = os.path.join(args.build_dir, project_desc["app_elf"])
+        if not os.path.exists(elf_file):
+            raise FatalError("ELF file not found. You need to build & flash the project before running debug targets", ctx)
+        gdb = project_desc["monitor_toolprefix"] + "gdb"
+        local_dir = project_desc["build_dir"]
+        gdbinit = os.path.join(local_dir, 'gdbinit')
+        create_local_gdbinit(gdbinit, elf_file)
+        args = [gdb, '-x={}'.format(gdbinit)]
+        if gdb_tui is not None:
+            args += ['-tui']
+        t = Thread(target=run_gdb, args=(args, ))
+        t.start()
+        while True:
+            try:
+                t.join()
+                break
+            except KeyboardInterrupt:
+                # Catching Keyboard interrupt, as this is used for breaking running program in gdb
+                continue
+            finally:
+                watch_openocd.join()
+                processes["threads_to_join"].remove(watch_openocd)
+
+    fail_if_openocd_failed = {
+        "names": ["--require-openocd", "--require_openocd"],
+        "help":
+        ("Fail this target if openocd (this targets dependency) failed.\n"),
+        "is_flag": True,
+        "default": False,
+    }
+    debug_actions = {
+        "global_action_callbacks": [global_callback],
+        "actions": {
+            "openocd": {
+                "callback": openocd,
+                "help": "Run openocd from current path",
+                "options": [
+                    {
+                        "names": ["--openocd-scripts", "--openocd_scripts"],
+                        "help":
+                        ("Script directory for openocd cfg files.\n"),
+                        "default":
+                        None,
+                    },
+                    {
+                        "names": ["--openocd-commands", "--openocd_commands"],
+                        "help":
+                        ("Command line arguments for openocd.\n"),
+                        "default": None,
+                    }
+                ],
+                "order_dependencies": ["all", "flash"],
+            },
+            "gdb": {
+                "callback": gdb,
+                "help": "Run the GDB.",
+                "options": [
+                    {
+                        "names": ["--gdb-tui", "--gdb_tui"],
+                        "help":
+                        ("run gdb in TUI mode\n"),
+                        "default":
+                        None,
+                    }, fail_if_openocd_failed
+                ],
+                "order_dependencies": ["all", "flash"],
+            },
+            "gdbgui": {
+                "callback": gdbui,
+                "help": "GDB UI in default browser.",
+                "options": [
+                    {
+                        "names": ["--gdbgui-port", "--gdbgui_port"],
+                        "help":
+                        ("The port on which gdbgui will be hosted. Default: 5000\n"),
+                        "default":
+                        None,
+                    }, fail_if_openocd_failed
+                ],
+                "order_dependencies": ["all", "flash"],
+            },
+            "gdbtui": {
+                "callback": gdbtui,
+                "help": "GDB TUI mode.",
+                "options": [fail_if_openocd_failed],
+                "order_dependencies": ["all", "flash"],
+            },
+            "post_debug": {
+                "callback": post_debug,
+                "help": "Utility target to read the output of async debug action and stop them.",
+                "options": [
+                    {
+                        "names": ["--block", "--block"],
+                        "help":
+                        ("Set to 1 for blocking the console on the outputs of async debug actions\n"),
+                        "default": 0,
+                    },
+                ],
+                "order_dependencies": [],
+            },
+        },
+    }
+
+    return debug_actions

tools/idf_py_actions/errors.py

@@ -2,5 +2,9 @@ class FatalError(RuntimeError):
     """
     Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build process.
     """
-
-    pass
+    def __init__(self, message, ctx=None):
+        super(RuntimeError, self).__init__(message)
+        # if context is defined, check for the cleanup tasks
+        if ctx is not None and "cleanup" in ctx.meta:
+            # cleans up the environment before failure
+            ctx.meta["cleanup"]()

tools/idf_py_actions/serial_ext.py

@@ -78,7 +78,7 @@ def action_extensions(base_actions, project_path):
         if not os.path.exists(elf_file):
             raise FatalError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', "
                              "and the binary on the device must match the one in the build directory exactly. "
-                             "Try '%s flash monitor'." % (elf_file, ctx.info_name))
+                             "Try '%s flash monitor'." % (elf_file, ctx.info_name), ctx)
         idf_monitor = os.path.join(os.environ["IDF_PATH"], "tools/idf_monitor.py")
         monitor_args = [PYTHON, idf_monitor]
         if args.port is not None:

tools/windows/windows_install_prerequisites.sh

@@ -34,7 +34,7 @@ set -e
 pacman --noconfirm -Syu # This step may require the terminal to be closed and restarted
 
 pacman --noconfirm -S --needed gettext-devel gcc git make ncurses-devel flex bison gperf vim \
-       mingw-w64-i686-python-pip mingw-w64-i686-python-cryptography unzip winpty
+       mingw-w64-i686-python-pip mingw-w64-i686-python-cryptography unzip winpty mingw-w64-i686-gcc
 
 # if IDF_PATH is set, install requirements now as well
 if [ -n "$IDF_PATH" ]; then