esp-idf/tools/build_apps.py

#!/usr/bin/env python

# coding=utf-8
#
# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
#

import argparse
import logging
import os.path
import re
import sys
from typing import List, Optional, TextIO

from find_build_apps import BUILD_SYSTEMS, BuildError, BuildItem, setup_logging
from find_build_apps.common import SIZE_JSON_FN, rmdir

# This RE will match GCC errors and many other fatal build errors and warnings as well
LOG_ERROR_WARNING = re.compile(r'(error|warning):', re.IGNORECASE)

# Log this many trailing lines from a failed build log, also
LOG_DEBUG_LINES = 25


def build_apps(
    build_items: List[BuildItem],
    parallel_count: int = 1,
    parallel_index: int = 1,
    dry_run: bool = False,
    build_verbose: bool = False,
    keep_going: bool = False,
    output_build_list: Optional[TextIO] = None,
    size_info: Optional[TextIO] = None
) -> None:
    if not build_items:
        logging.warning('Empty build list')
        sys.exit(0)

    num_builds = len(build_items)
    num_jobs = parallel_count
    job_index = parallel_index - 1  # convert to 0-based index
    num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
    min_job_index = num_builds_per_job * job_index
    if min_job_index >= num_builds:
        logging.warning(
            f'Nothing to do for job {job_index + 1} (build total: {num_builds}, per job: {num_builds_per_job})')
        sys.exit(0)

    max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1)
    logging.info('Total {} builds, max. {} builds per job, running builds {}-{}'.format(
        num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1))

    builds_for_current_job = build_items[min_job_index:max_job_index + 1]
    for i, build_item in enumerate(builds_for_current_job):
        index = i + min_job_index + 1
        build_item.index = index
        build_item.dry_run = dry_run
        build_item.verbose = build_verbose
        build_item.keep_going = keep_going
        logging.debug('\tBuild {}: {}'.format(index, repr(build_item)))
        if output_build_list:
            output_build_list.write(build_item.to_json_expanded() + '\n')

    failed_builds = []
    for build_item in builds_for_current_job:
        logging.info('Running build {}: {}'.format(build_item.index, repr(build_item)))
        build_system_class = BUILD_SYSTEMS[build_item.build_system]
        try:
            build_system_class.build(build_item)
        except BuildError as e:
            logging.error(str(e))
            if build_item.build_log_path:
                log_filename = os.path.basename(build_item.build_log_path)
                with open(build_item.build_log_path, 'r') as f:
                    lines = [line.rstrip() for line in f.readlines() if line.rstrip()]  # non-empty lines
                    logging.debug('Error and warning lines from {}:'.format(log_filename))
                    for line in lines:
                        if LOG_ERROR_WARNING.search(line):
                            logging.warning('>>> {}'.format(line))
                    logging.debug('Last {} lines of {}:'.format(LOG_DEBUG_LINES, log_filename))
                    for line in lines[-LOG_DEBUG_LINES:]:
                        logging.debug('>>> {}'.format(line))
            if keep_going:
                failed_builds.append(build_item)
            else:
                sys.exit(1)
        else:
            if size_info:
                build_item.write_size_info(size_info)
            if not build_item.preserve:
                logging.info(f'Removing build directory {build_item.build_path}')
                # we only remove binaries here, log files are still needed by check_build_warnings.py
                rmdir(build_item.build_path, exclude_file_pattern=SIZE_JSON_FN)

    if failed_builds:
        logging.error('The following build have failed:')
        for build in failed_builds:
            logging.error('\t{}'.format(build))
        sys.exit(1)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='ESP-IDF app builder')
    parser.add_argument(
        '-v',
        '--verbose',
        action='count',
        help='Increase the logging level of the script. Can be specified multiple times.',
    )
    parser.add_argument(
        '--build-verbose',
        action='store_true',
        help='Enable verbose output from build system.',
    )
    parser.add_argument(
        '--log-file',
        type=argparse.FileType('w'),
        help='Write the script log to the specified file, instead of stderr',
    )
    parser.add_argument(
        '--parallel-count',
        default=1,
        type=int,
        help="Number of parallel build jobs. Note that this script doesn't start the jobs, " +
             'it needs to be executed multiple times with same value of --parallel-count and ' +
             'different values of --parallel-index.',
    )
    parser.add_argument(
        '--parallel-index',
        default=1,
        type=int,
        help='Index (1-based) of the job, out of the number specified by --parallel-count.',
    )
    parser.add_argument(
        '--format',
        default='json',
        choices=['json'],
        help='Format to read the list of builds',
    )
    parser.add_argument(
        '--dry-run',
        action='store_true',
        help="Don't actually build, only print the build commands",
    )
    parser.add_argument(
        '--keep-going',
        action='store_true',
        help="Don't exit immediately when a build fails.",
    )
    parser.add_argument(
        '--output-build-list',
        type=argparse.FileType('w'),
        help='If specified, the list of builds (with all the placeholders expanded) will be written to this file.',
    )
    parser.add_argument(
        '--size-info',
        type=argparse.FileType('a'),
        help='If specified, the test case name and size info json will be written to this file'
    )
    parser.add_argument(
        'build_list',
        type=argparse.FileType('r'),
        nargs='?',
        default=sys.stdin,
        help='Name of the file to read the list of builds from. If not specified, read from stdin.',
    )
    args = parser.parse_args()
    setup_logging(args)
    items = [BuildItem.from_json(line) for line in args.build_list]
    build_apps(items, args.parallel_count, args.parallel_index, args.dry_run, args.build_verbose,
               args.keep_going, args.output_build_list, args.size_info)