esp-idf/tools/ci/python_packages/idf_iperf_test_util/TestReport.py

# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""
this module generates markdown format test report for throughput test.

The test report contains 2 parts:

1. throughput with different configs
2. throughput with RSSI
"""
import os


class ThroughputForConfigsReport(object):
    THROUGHPUT_TYPES = ['tcp_tx', 'tcp_rx', 'udp_tx', 'udp_rx']

    REPORT_FILE_NAME = 'ThroughputForConfigs.md'

    def __init__(self, output_path, ap_ssid, throughput_results, sdkconfig_files):
        """
        :param ap_ssid: the ap we expected to use
        :param throughput_results: config with the following type::
            {
                "config_name": {
                    "tcp_tx": result,
                    "tcp_rx": result,
                    "udp_tx": result,
                    "udp_rx": result,
                },
                "config_name2": {},
            }
        """
        self.output_path = output_path
        self.ap_ssid = ap_ssid
        self.results = throughput_results
        self.sdkconfigs = dict()
        for config_name in sdkconfig_files:
            self.sdkconfigs[config_name] = self._parse_config_file(sdkconfig_files[config_name])
        if not os.path.exists(output_path):
            os.makedirs(output_path)
        self.sort_order = list(self.sdkconfigs.keys())
        self.sort_order.sort()

    @staticmethod
    def _parse_config_file(config_file_path):
        sdkconfig = {}
        with open(config_file_path, 'r') as f:
            for line in f:
                if not line.isspace():
                    if line[0] == '#':
                        continue
                    name, value = line.split('=')
                    value = value.strip('\r\n')
                    sdkconfig[name] = value if value else 'n'
        return sdkconfig

    def _generate_the_difference_between_configs(self):
        """
        generate markdown list for different configs::

            default: esp-idf default
            low:
                * `config name 1`: old value -> new value
                * `config name 2`: old value -> new value
                * ...
            ...

        """

        data = '## Config Definition:\r\n\r\n'

        def find_difference(base, new):
            _difference = {}
            all_configs = set(base.keys())
            all_configs.update(set(new.keys()))
            for _config in all_configs:
                try:
                    _base_value = base[_config]
                except KeyError:
                    _base_value = 'null'
                try:
                    _new_value = new[_config]
                except KeyError:
                    _new_value = 'null'
                if _base_value != _new_value:
                    _difference[_config] = '{} -> {}'.format(_base_value, _new_value)
            return _difference

        for i, _config_name in enumerate(self.sort_order):
            current_config = self.sdkconfigs[_config_name]

            if i > 0:
                previous_config_name = self.sort_order[i - 1]
                previous_config = self.sdkconfigs[previous_config_name]
            else:
                previous_config = previous_config_name = None

            if previous_config:
                # log the difference
                difference = find_difference(previous_config, current_config)
                data += '* {} (compared to {}):\r\n'.format(_config_name, previous_config_name)
                for diff_name in difference:
                    data += '    * `{}`: {}\r\n'.format(diff_name, difference[diff_name])
        return data

    def _generate_report_for_one_type(self, throughput_type):
        """
        generate markdown table with the following format::

            | config name | throughput (Mbps) | free heap size (bytes) |
            |-------------|-------------------|------------------------|
            | default     | 32.11             | 147500                 |
            | low         | 32.11             | 147000                 |
            | medium      | 33.22             | 120000                 |
            | high        | 43.11             | 100000                 |
            | max         | 45.22             | 79000                  |
        """
        empty = True

        ret = '\r\n### {} {}\r\n\r\n'.format(*throughput_type.split('_'))
        ret += '| config name | throughput (Mbps) | free heap size (bytes) |\r\n'
        ret += '|-------------|-------------------|------------------------|\r\n'
        for config in self.sort_order:
            try:
                result = self.results[config][throughput_type]
                throughput = '{:.02f}'.format(max(result.throughput_by_att[self.ap_ssid].values()))
                heap_size = str(result.heap_size)
                # although markdown table will do alignment
                # do align here for better text editor presentation
                ret += '| {:<12}| {:<18}| {:<23}|\r\n'.format(config, throughput, heap_size)
                empty = False
            except KeyError:
                pass
        return ret if not empty else ''

    def generate_report(self):
        data = '# Throughput for different configs\r\n'
        data += '\r\nAP: {}\r\n'.format(self.ap_ssid)

        for throughput_type in self.THROUGHPUT_TYPES:
            data += self._generate_report_for_one_type(throughput_type)
            data += '\r\n------\r\n'

        data += self._generate_the_difference_between_configs()

        with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), 'w') as f:
            f.write(data)


class ThroughputVsRssiReport(object):

    REPORT_FILE_NAME = 'ThroughputVsRssi.md'

    def __init__(self, output_path, throughput_results):
        """
        :param throughput_results: config with the following type::
            {
                "tcp_tx": result,
                "tcp_rx": result,
                "udp_tx": result,
                "udp_rx": result,
            }
        """
        self.output_path = output_path
        self.raw_data_path = os.path.join(output_path, 'raw_data')
        self.results = throughput_results
        self.throughput_types = list(self.results.keys())
        self.throughput_types.sort()
        if not os.path.exists(self.raw_data_path):
            os.makedirs(self.raw_data_path)

    def _generate_summary(self):
        """
        generate summary with the following format::

            | item    | curve analysis | max throughput (Mbps) |
            |---------|----------------|-----------------------|
            | tcp tx  | Success        | 32.11                 |
            | tcp rx  | Success        | 32.11                 |
            | udp tx  | Success        | 45.22                 |
            | udp rx  | Failed         | 55.44                 |
        """

        ret = '\r\n### Summary\r\n\r\n'
        ret += '| item    | curve analysis | max throughput (Mbps) |\r\n'
        ret += '|---------|----------------|-----------------------|\r\n'

        for _type in self.throughput_types:
            result = self.results[_type]
            max_throughput = 0.0
            curve_analysis = 'Failed' if result.error_list else 'Success'
            for ap_ssid in result.throughput_by_att:
                _max_for_ap = max(result.throughput_by_rssi[ap_ssid].values())
                if _max_for_ap > max_throughput:
                    max_throughput = _max_for_ap
            max_throughput = '{:.02f}'.format(max_throughput)
            ret += '| {:<8}| {:<15}| {:<22}|\r\n'.format('{}_{}'.format(result.proto, result.direction),
                                                         curve_analysis, max_throughput)
        return ret

    def _generate_report_for_one_type(self, result):
        """
        generate markdown table with the following format::

            ### tcp rx

            Errors:

            * detected error 1
            * ...

            AP: ap_ssid

            ![throughput Vs RSSI](path to figure)

            AP: ap_ssid

            ![throughput Vs RSSI](path to figure)

        """
        result.post_analysis()
        ret = '\r\n### {} {}\r\n'.format(result.proto, result.direction)
        if result.error_list:
            ret += '\r\nErrors:\r\n\r\n'
            for error in result.error_list:
                ret += '* ' + error + '\r\n'
        for ap_ssid in result.throughput_by_rssi:
            ret += '\r\nAP: {}\r\n'.format(ap_ssid)
            # draw figure
            file_name = result.draw_throughput_figure(self.raw_data_path, ap_ssid, 'rssi')
            result.draw_throughput_figure(self.raw_data_path, ap_ssid, 'att')
            result.draw_rssi_vs_att_figure(self.raw_data_path, ap_ssid)

            ret += '\r\n[throughput Vs RSSI]({})\r\n'.format(os.path.join('raw_data', file_name))

        return ret

    def generate_report(self):
        data = '# Throughput Vs RSSI\r\n'

        data += self._generate_summary()

        for _type in self.throughput_types:
            data += self._generate_report_for_one_type(self.results[_type])

        with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), 'w') as f:
            f.write(data)