example test: add test case for iperf example

components/idf_test/include/idf_performance.h

@@ -17,4 +17,8 @@
 #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING                            30
 #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_NO_DMA                     27
 #define IDF_PERFORMANCE_MAX_VFS_OPEN_WRITE_CLOSE_TIME                           20000
-
+// throughput performance by iperf
+#define IDF_PERFORMANCE_MIN_TCP_RX_THROUGHPUT                                   50
+#define IDF_PERFORMANCE_MIN_TCP_TX_THROUGHPUT                                   40
+#define IDF_PERFORMANCE_MIN_UDP_RX_THROUGHPUT                                   80
+#define IDF_PERFORMANCE_MIN_UDP_TX_THROUGHPUT                                   50

examples/wifi/iperf/iperf_test.py

@@ -0,0 +1,663 @@
+"""
+Test case for iperf example.
+
+This test case might have problem running on windows:
+
+1. direct use of `make`
+2. use `sudo killall iperf` to force kill iperf, didn't implement windows version
+
+The test env Example_ShieldBox do need the following config::
+
+  Example_ShieldBox:
+    ap_list:
+      - ssid: "ssid"
+        password: "password"
+        outlet: 1
+    apc_ip: "192.168.1.88"
+    attenuator_port: "/dev/ttyUSB0"
+    iperf: "/dev/ttyUSB1"
+    apc_ip: "192.168.1.88"
+    pc_nic: "eth0"
+"""
+import re
+import os
+import sys
+import time
+import subprocess
+
+# add current folder to system path for importing test_report
+sys.path.append(os.path.dirname(__file__))
+# this is a test case write with tiny-test-fw.
+# to run test cases outside tiny-test-fw,
+# we need to set environment variable `TEST_FW_PATH`,
+# then get and insert `TEST_FW_PATH` to sys path before import FW module
+test_fw_path = os.getenv("TEST_FW_PATH")
+if test_fw_path and test_fw_path not in sys.path:
+    sys.path.insert(0, test_fw_path)
+
+import IDF
+import DUT
+import Utility
+from Utility import (Attenuator, PowerControl, LineChart)
+from test_report import (ThroughputForConfigsReport, ThroughputVsRssiReport)
+
+# configurations
+TEST_TIME = TEST_TIMEOUT = 60
+WAIT_AP_POWER_ON_TIMEOUT = 90
+SCAN_TIMEOUT = 3
+SCAN_RETRY_COUNT = 3
+RETRY_COUNT_FOR_BEST_PERFORMANCE = 2
+ATTEN_VALUE_LIST = range(0, 60, 2)
+
+# constants
+FAILED_TO_SCAN_RSSI = -97
+INVALID_HEAP_SIZE = 0xFFFFFFFF
+
+PC_IPERF_TEMP_LOG_FILE = ".tmp_iperf.log"
+CONFIG_NAME_PATTERN = re.compile(r"sdkconfig\.defaults\.(.+)")
+
+# We need to auto compare the difference between adjacent configs (01 -> 00, 02 -> 01, ...) and put them to reports.
+# Using numbers for config will make this easy.
+# Use default value `99` for config with best performance.
+BEST_PERFORMANCE_CONFIG = "99"
+
+
+class TestResult(object):
+    """ record, analysis test result and convert data to output format """
+
+    PC_BANDWIDTH_LOG_PATTERN = re.compile(r"(\d+).0\s*-\s*(\d+).0\s+sec\s+[\d.]+\s+MBytes\s+([\d.]+)\s+Mbits/sec")
+    DUT_BANDWIDTH_LOG_PATTERN = re.compile(r"(\d+)-\s+(\d+)\s+sec\s+([\d.]+)\s+Mbits/sec")
+
+    ZERO_POINT_THRESHOLD = -88  # RSSI, dbm
+    ZERO_THROUGHPUT_THRESHOLD = -92  # RSSI, dbm
+    BAD_POINT_RSSI_THRESHOLD = -85  # RSSI, dbm
+    BAD_POINT_MIN_THRESHOLD = 3  # Mbps
+    BAD_POINT_PERCENTAGE_THRESHOLD = 0.3
+
+    # we need at least 1/2 valid points to qualify the test result
+    THROUGHPUT_QUALIFY_COUNT = TEST_TIME / 2
+
+    def __init__(self, proto, direction, config_name):
+        self.proto = proto
+        self.direction = direction
+        self.config_name = config_name
+        self.throughput_by_rssi = dict()
+        self.throughput_by_att = dict()
+        self.att_rssi_map = dict()
+        self.heap_size = INVALID_HEAP_SIZE
+        self.error_list = []
+
+    def _save_result(self, throughput, ap_ssid, att, rssi, heap_size):
+        """
+        save the test results:
+
+        * record the better throughput if att/rssi is the same.
+        * record the min heap size.
+        """
+        if ap_ssid not in self.att_rssi_map:
+            # for new ap, create empty dict()
+            self.throughput_by_att[ap_ssid] = dict()
+            self.throughput_by_rssi[ap_ssid] = dict()
+            self.att_rssi_map[ap_ssid] = dict()
+
+        self.att_rssi_map[ap_ssid][att] = rssi
+
+        def record_throughput(database, key_value):
+            try:
+                # we save the larger value for same att
+                if throughput > database[ap_ssid][key_value]:
+                    database[ap_ssid][key_value] = throughput
+            except KeyError:
+                database[ap_ssid][key_value] = throughput
+
+        record_throughput(self.throughput_by_att, att)
+        record_throughput(self.throughput_by_rssi, rssi)
+
+        if int(heap_size) < self.heap_size:
+            self.heap_size = int(heap_size)
+
+    def add_result(self, raw_data, ap_ssid, att, rssi, heap_size):
+        """
+        add result for one test
+
+        :param raw_data: iperf raw data
+        :param ap_ssid: ap ssid that tested
+        :param att: attenuate value
+        :param rssi: AP RSSI
+        :param heap_size: min heap size during test
+        :return: throughput
+        """
+        fall_to_0_recorded = 0
+        throughput_list = []
+        result_list = self.PC_BANDWIDTH_LOG_PATTERN.findall(raw_data)
+        if not result_list:
+            # failed to find raw data by PC pattern, it might be DUT pattern
+            result_list = self.DUT_BANDWIDTH_LOG_PATTERN.findall(raw_data)
+
+        for result in result_list:
+            if int(result[1]) - int(result[0]) != 1:
+                # this could be summary, ignore this
+                continue
+            throughput_list.append(float(result[2]))
+            if float(result[2]) == 0 and rssi > self.ZERO_POINT_THRESHOLD \
+                    and fall_to_0_recorded < 1:
+                # throughput fall to 0 error. we only record 1 records for one test
+                self.error_list.append("[Error][fall to 0][{}][att: {}][rssi: {}]: 0 throughput interval: {}-{}"
+                                       .format(ap_ssid, att, rssi, result[0], result[1]))
+                fall_to_0_recorded += 1
+
+        if len(throughput_list) > self.THROUGHPUT_QUALIFY_COUNT:
+            throughput = sum(throughput_list) / len(throughput_list)
+        else:
+            throughput = 0.0
+
+        if throughput == 0 and rssi > self.ZERO_THROUGHPUT_THRESHOLD:
+                self.error_list.append("[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found"
+                                       .format(ap_ssid, att, rssi))
+
+        self._save_result(throughput, ap_ssid, att, rssi, heap_size)
+
+        return throughput
+
+    def post_analysis(self):
+        """
+        some rules need to be checked after we collected all test raw data:
+
+        1. throughput value 30% worse than the next point with lower RSSI
+        2. throughput value 30% worse than the next point with larger attenuate
+        """
+        def analysis_bad_point(data, index_type):
+            for ap_ssid in data:
+                result_dict = data[ap_ssid]
+                index_list = result_dict.keys()
+                index_list.sort()
+                if index_type == "att":
+                    index_list.reverse()
+
+                for i, index_value in enumerate(index_list[1:]):
+                    if index_value < self.BAD_POINT_RSSI_THRESHOLD or \
+                            result_dict[index_list[i]] < self.BAD_POINT_MIN_THRESHOLD:
+                        continue
+                    _percentage = result_dict[index_value] / result_dict[index_list[i]]
+                    if _percentage < 1 - self.BAD_POINT_PERCENTAGE_THRESHOLD:
+                        self.error_list.append("[Error][Bad point][{}][{}: {}]: drop {:.02f}%"
+                                               .format(ap_ssid, index_type, index_value,
+                                                       (1 - _percentage) * 100))
+
+        analysis_bad_point(self.throughput_by_rssi, "rssi")
+        analysis_bad_point(self.throughput_by_att, "att")
+
+    @staticmethod
+    def _convert_to_draw_format(data, label):
+        keys = data.keys()
+        keys.sort()
+        return {
+            "x-axis": keys,
+            "y-axis": [data[x] for x in keys],
+            "label": label,
+        }
+
+    def draw_throughput_figure(self, path, ap_ssid, draw_type):
+        """
+        :param path: folder to save figure. make sure the folder is already created.
+        :param ap_ssid: ap ssid string or a list of ap ssid string
+        :param draw_type: "att" or "rssi"
+        :return: file_name
+        """
+        if draw_type == "rssi":
+            type_name = "RSSI"
+            data = self.throughput_by_rssi
+        elif draw_type == "att":
+            type_name = "Att"
+            data = self.throughput_by_att
+        else:
+            raise AssertionError("draw type not supported")
+        if isinstance(ap_ssid, list):
+            file_name = "ThroughputVs{}_{}_{}_{}.png".format(type_name, self.proto, self.direction,
+                                                             hash(ap_ssid)[:6])
+            data_list = [self._convert_to_draw_format(data[_ap_ssid], _ap_ssid)
+                         for _ap_ssid in ap_ssid]
+        else:
+            file_name = "ThroughputVs{}_{}_{}_{}.png".format(type_name, self.proto, self.direction, ap_ssid)
+            data_list = [self._convert_to_draw_format(data[ap_ssid], ap_ssid)]
+
+        LineChart.draw_line_chart(os.path.join(path, file_name),
+                                  "Throughput Vs {} ({} {})".format(type_name, self.proto, self.direction),
+                                  "Throughput (Mbps)",
+                                  "{} (dbm)".format(type_name),
+                                  data_list)
+        return file_name
+
+    def draw_rssi_vs_att_figure(self, path, ap_ssid):
+        """
+        :param path: folder to save figure. make sure the folder is already created.
+        :param ap_ssid: ap to use
+        :return: file_name
+        """
+        if isinstance(ap_ssid, list):
+            file_name = "AttVsRSSI_{}.png".format(hash(ap_ssid)[:6])
+            data_list = [self._convert_to_draw_format(self.att_rssi_map[_ap_ssid], _ap_ssid)
+                         for _ap_ssid in ap_ssid]
+        else:
+            file_name = "AttVsRSSI_{}.png".format(ap_ssid)
+            data_list = [self._convert_to_draw_format(self.att_rssi_map[ap_ssid], ap_ssid)]
+        LineChart.draw_line_chart(os.path.join(path, file_name),
+                                  "Att Vs RSSI",
+                                  "Att (dbm)",
+                                  "RSSI (dbm)",
+                                  data_list)
+        return file_name
+
+    def get_best_throughput(self):
+        """  get the best throughput during test """
+        best_for_aps = [max(self.throughput_by_att[ap_ssid].values())
+                        for ap_ssid in self.throughput_by_att]
+        return max(best_for_aps)
+
+    def __str__(self):
+        """
+        returns summary for this test:
+
+        1. test result (success or fail)
+        2. best performance for each AP
+        3. min free heap size during test
+        """
+        if self.throughput_by_att:
+            ret = "[{}_{}][{}]: {}\r\n\r\n".format(self.proto, self.direction, self.config_name,
+                                                   "Fail" if self.error_list else "Success")
+            ret += "Performance for each AP:\r\n"
+            for ap_ssid in self.throughput_by_att:
+                ret += "[{}]: {:.02f} Mbps\r\n".format(ap_ssid, max(self.throughput_by_att[ap_ssid].values()))
+            if self.heap_size != INVALID_HEAP_SIZE:
+                ret += "Minimum heap size: {}".format(self.heap_size)
+        else:
+            ret = ""
+        return ret
+
+
+class IperfTestUtility(object):
+    """ iperf test implementation """
+
+    def __init__(self, dut, config_name, ap_ssid, ap_password,
+                 pc_nic_ip, pc_iperf_log_file, test_result=None):
+        self.config_name = config_name
+        self.dut = dut
+
+        self.pc_iperf_log_file = pc_iperf_log_file
+        self.ap_ssid = ap_ssid
+        self.ap_password = ap_password
+        self.pc_nic_ip = pc_nic_ip
+
+        if test_result:
+            self.test_result = test_result
+        else:
+            self.test_result = {
+                "tcp_tx": TestResult("tcp", "tx", config_name),
+                "tcp_rx": TestResult("tcp", "rx", config_name),
+                "udp_tx": TestResult("udp", "tx", config_name),
+                "udp_rx": TestResult("udp", "rx", config_name),
+            }
+
+    def setup(self):
+        """
+        setup iperf test:
+
+        1. kill current iperf process
+        2. reboot DUT (currently iperf is not very robust, need to reboot DUT)
+        3. scan to get AP RSSI
+        4. connect to AP
+        """
+        try:
+            subprocess.check_output("sudo killall iperf 2>&1 > /dev/null", shell=True)
+        except subprocess.CalledProcessError:
+            pass
+        self.dut.write("restart")
+        self.dut.expect("esp32>")
+        self.dut.write("scan {}".format(self.ap_ssid))
+        for _ in range(SCAN_RETRY_COUNT):
+            try:
+                rssi = int(self.dut.expect(re.compile(r"\[{}]\[rssi=(-\d+)]".format(self.ap_ssid)),
+                                           timeout=SCAN_TIMEOUT)[0])
+                break
+            except DUT.ExpectTimeout:
+                continue
+        else:
+            raise AssertionError("Failed to scan AP")
+        self.dut.write("sta {} {}".format(self.ap_ssid, self.ap_password))
+        dut_ip = self.dut.expect(re.compile(r"event: sta ip: ([\d.]+), mask: ([\d.]+), gw: ([\d.]+)"))[0]
+        return dut_ip, rssi
+
+    def _save_test_result(self, test_case, raw_data, att, rssi, heap_size):
+        return self.test_result[test_case].add_result(raw_data, self.ap_ssid, att, rssi, heap_size)
+
+    def _test_once(self, proto, direction):
+        """ do measure once for one type """
+        # connect and scan to get RSSI
+        dut_ip, rssi = self.setup()
+
+        assert direction in ["rx", "tx"]
+        assert proto in ["tcp", "udp"]
+
+        # run iperf test
+        if direction == "tx":
+            with open(PC_IPERF_TEMP_LOG_FILE, "w") as f:
+                if proto == "tcp":
+                    process = subprocess.Popen(["iperf", "-s", "-B", self.pc_nic_ip,
+                                                "-t", str(TEST_TIME), "-i", "1", "-f", "m"],
+                                               stdout=f, stderr=f)
+                    self.dut.write("iperf -c {} -i 1 -t {}".format(self.pc_nic_ip, TEST_TIME))
+                else:
+                    process = subprocess.Popen(["iperf", "-s", "-u", "-B", self.pc_nic_ip,
+                                                "-t", str(TEST_TIME), "-i", "1", "-f", "m"],
+                                               stdout=f, stderr=f)
+                    self.dut.write("iperf -c {} -u -i 1 -t {}".format(self.pc_nic_ip, TEST_TIME))
+
+                for _ in range(TEST_TIMEOUT):
+                    if process.poll() is not None:
+                        break
+                    time.sleep(1)
+                else:
+                    process.terminate()
+
+            with open(PC_IPERF_TEMP_LOG_FILE, "r") as f:
+                pc_raw_data = server_raw_data = f.read()
+        else:
+            with open(PC_IPERF_TEMP_LOG_FILE, "w") as f:
+                if proto == "tcp":
+                    self.dut.write("iperf -s -i 1 -t {}".format(TEST_TIME))
+                    process = subprocess.Popen(["iperf", "-c", dut_ip,
+                                                "-t", str(TEST_TIME), "-f", "m"],
+                                               stdout=f, stderr=f)
+                else:
+                    self.dut.write("iperf -s -u -i 1 -t {}".format(TEST_TIME))
+                    process = subprocess.Popen(["iperf", "-c", dut_ip, "-u", "-b", "100M",
+                                                "-t", str(TEST_TIME), "-f", "m"],
+                                               stdout=f, stderr=f)
+
+                for _ in range(TEST_TIMEOUT):
+                    if process.poll() is not None:
+                        break
+                    time.sleep(1)
+                else:
+                    process.terminate()
+
+            server_raw_data = self.dut.read()
+            with open(PC_IPERF_TEMP_LOG_FILE, "r") as f:
+                pc_raw_data = f.read()
+
+        # save PC iperf logs to console
+        with open(self.pc_iperf_log_file, "a+") as f:
+            f.write("## [{}] `{}`\r\n##### {}"
+                    .format(self.config_name,
+                            "{}_{}".format(proto, direction),
+                            time.strftime("%m-%d %H:%M:%S", time.localtime(time.time()))))
+            f.write('\r\n```\r\n\r\n' + pc_raw_data + '\r\n```\r\n')
+        self.dut.write("heap")
+        heap_size = self.dut.expect(re.compile(r"min heap size: (\d+)\D"))[0]
+
+        # return server raw data (for parsing test results) and RSSI
+        return server_raw_data, rssi, heap_size
+
+    def run_test(self, proto, direction, atten_val):
+        """
+        run test for one type, with specified atten_value and save the test result
+
+        :param proto: tcp or udp
+        :param direction: tx or rx
+        :param atten_val: attenuate value
+        """
+        rssi = FAILED_TO_SCAN_RSSI
+        heap_size = INVALID_HEAP_SIZE
+        try:
+            server_raw_data, rssi, heap_size = self._test_once(proto, direction)
+            throughput = self._save_test_result("{}_{}".format(proto, direction),
+                                                server_raw_data, atten_val,
+                                                rssi, heap_size)
+            Utility.console_log("[{}][{}_{}][{}][{}]: {:.02f}"
+                                .format(self.config_name, proto, direction, rssi, self.ap_ssid, throughput))
+        except Exception as e:
+            self._save_test_result("{}_{}".format(proto, direction), "", atten_val, rssi, heap_size)
+            Utility.console_log("Failed during test: {}".format(e))
+
+    def run_all_cases(self, atten_val):
+        """
+        run test for all types (udp_tx, udp_rx, tcp_tx, tcp_rx).
+
+        :param atten_val: attenuate value
+        """
+        self.run_test("tcp", "tx", atten_val)
+        self.run_test("tcp", "rx", atten_val)
+        self.run_test("udp", "tx", atten_val)
+        self.run_test("udp", "rx", atten_val)
+
+    def wait_ap_power_on(self):
+        """
+        AP need to take sometime to power on. It changes for different APs.
+        This method will scan to check if the AP powers on.
+
+        :return: True or False
+        """
+        self.dut.write("restart")
+        self.dut.expect("esp32>")
+        for _ in range(WAIT_AP_POWER_ON_TIMEOUT // SCAN_TIMEOUT):
+            try:
+                self.dut.write("scan {}".format(self.ap_ssid))
+                self.dut.expect(re.compile(r"\[{}]\[rssi=(-\d+)]".format(self.ap_ssid)),
+                                timeout=SCAN_TIMEOUT)
+                ret = True
+                break
+            except DUT.ExpectTimeout:
+                pass
+        else:
+            ret = False
+        return ret
+
+
+def build_iperf_with_config(config_name):
+    """
+    we need to build iperf example with different configurations.
+
+    :param config_name: sdkconfig we want to build
+    """
+
+    # switch to iperf example path before build when we're running test with Runner
+    example_path = os.path.dirname(__file__)
+    cwd = os.getcwd()
+    if cwd != example_path and example_path:
+        os.chdir(example_path)
+    try:
+        subprocess.check_call("make clean > /dev/null", shell=True)
+        subprocess.check_call(["cp", "sdkconfig.defaults.{}".format(config_name), "sdkconfig.defaults"])
+        subprocess.check_call(["rm", "-f", "sdkconfig"])
+        subprocess.check_call("make defconfig > /dev/null", shell=True)
+        # save sdkconfig to generate config comparision report
+        subprocess.check_call(["cp", "sdkconfig", "sdkconfig.{}".format(config_name)])
+        subprocess.check_call("make -j5 > /dev/null", shell=True)
+        subprocess.check_call("make print_flash_cmd | tail -n 1 > build/download.config", shell=True)
+    finally:
+        os.chdir(cwd)
+
+
+def get_configs(env):
+    att_port = env.get_variable("attenuator_port")
+    ap_list = env.get_variable("ap_list")
+    pc_nic_ip = env.get_pc_nic_info("pc_nic", "ipv4")["addr"]
+    apc_ip = env.get_variable("apc_ip")
+    pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md")
+    return att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file
+
+
+@IDF.idf_example_test(env_tag="Example_ShieldBox", category="stress")
+def test_wifi_throughput_with_different_configs(env, extra_data):
+    """
+    steps: |
+      1. build iperf with specified configs
+      2. test throughput for all routers
+    """
+    att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env)
+    ap_info = ap_list[0]
+
+    config_names_raw = subprocess.check_output(["ls", os.path.dirname(os.path.abspath(__file__))])
+
+    test_result = dict()
+    sdkconfig_files = dict()
+
+    for config_name in CONFIG_NAME_PATTERN.findall(config_names_raw):
+        # 1. build config
+        build_iperf_with_config(config_name)
+        sdkconfig_files[config_name] = os.path.join(os.path.dirname(__file__),
+                                                    "sdkconfig.{}".format(config_name))
+
+        # 2. get DUT and download
+        dut = env.get_dut("iperf", "examples/wifi/iperf")
+        dut.start_app()
+        dut.expect("esp32>")
+
+        # 3. run test for each required att value
+        test_result[config_name] = {
+            "tcp_tx": TestResult("tcp", "tx", config_name),
+            "tcp_rx": TestResult("tcp", "rx", config_name),
+            "udp_tx": TestResult("udp", "tx", config_name),
+            "udp_rx": TestResult("udp", "rx", config_name),
+        }
+
+        test_utility = IperfTestUtility(dut, config_name, ap_info["ssid"],
+                                        ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result[config_name])
+
+        PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF")
+        PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"})
+        assert Attenuator.set_att(att_port, 0) is True
+
+        if not test_utility.wait_ap_power_on():
+            Utility.console_log("[{}] failed to power on, skip testing this AP"
+                                .format(ap_info["ssid"]), color="red")
+
+        for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE):
+            test_utility.run_all_cases(0)
+
+        for result_type in test_result[config_name]:
+            summary = str(test_result[config_name][result_type])
+            if summary:
+                Utility.console_log(summary, color="orange")
+
+        # 4. check test results
+        env.close_dut("iperf")
+
+    # 5. generate report
+    report = ThroughputForConfigsReport(os.path.join(env.log_path, "ThroughputForConfigsReport"),
+                                        ap_info["ssid"], test_result, sdkconfig_files)
+    report.generate_report()
+
+
+@IDF.idf_example_test(env_tag="Example_ShieldBox", category="stress")
+def test_wifi_throughput_vs_rssi(env, extra_data):
+    """
+    steps: |
+      1. build with best performance config
+      2. switch on one router
+      3. set attenuator value from 0-60 for each router
+      4. test TCP tx rx and UDP tx rx throughput
+    """
+    att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env)
+
+    pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md")
+
+    test_result = {
+        "tcp_tx": TestResult("tcp", "tx", BEST_PERFORMANCE_CONFIG),
+        "tcp_rx": TestResult("tcp", "rx", BEST_PERFORMANCE_CONFIG),
+        "udp_tx": TestResult("udp", "tx", BEST_PERFORMANCE_CONFIG),
+        "udp_rx": TestResult("udp", "rx", BEST_PERFORMANCE_CONFIG),
+    }
+
+    # 1. build config
+    build_iperf_with_config(BEST_PERFORMANCE_CONFIG)
+
+    # 2. get DUT and download
+    dut = env.get_dut("iperf", "examples/wifi/iperf")
+    dut.start_app()
+    dut.expect("esp32>")
+
+    # 3. run test for each required att value
+    for ap_info in ap_list:
+        test_utility = IperfTestUtility(dut, BEST_PERFORMANCE_CONFIG, ap_info["ssid"], ap_info["password"],
+                                        pc_nic_ip, pc_iperf_log_file, test_result)
+
+        PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF")
+        PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"})
+        Attenuator.set_att(att_port, 0)
+
+        if not test_utility.wait_ap_power_on():
+            Utility.console_log("[{}] failed to power on, skip testing this AP"
+                                .format(ap_info["ssid"]), color="red")
+            continue
+
+        for atten_val in ATTEN_VALUE_LIST:
+            assert Attenuator.set_att(att_port, atten_val) is True
+            test_utility.run_all_cases(atten_val)
+
+    # 4. check test results
+    env.close_dut("iperf")
+
+    # 5. generate report
+    report = ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"),
+                                    test_result)
+    report.generate_report()
+
+
+@IDF.idf_example_test(env_tag="Example_ShieldBox")
+def test_wifi_throughput_basic(env, extra_data):
+    """
+    steps: |
+      1. test TCP tx rx and UDP tx rx throughput
+      2. compare with the pre-defined pass standard
+    """
+    att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env)
+    ap_info = ap_list[0]
+
+    # 1. build iperf with best config
+    build_iperf_with_config(BEST_PERFORMANCE_CONFIG)
+
+    # 2. get DUT
+    dut = env.get_dut("iperf", "examples/wifi/iperf")
+    dut.start_app()
+    dut.expect("esp32>")
+
+    # 3. preparing
+    test_result = {
+        "tcp_tx": TestResult("tcp", "tx", BEST_PERFORMANCE_CONFIG),
+        "tcp_rx": TestResult("tcp", "rx", BEST_PERFORMANCE_CONFIG),
+        "udp_tx": TestResult("udp", "tx", BEST_PERFORMANCE_CONFIG),
+        "udp_rx": TestResult("udp", "rx", BEST_PERFORMANCE_CONFIG),
+    }
+
+    test_utility = IperfTestUtility(dut, BEST_PERFORMANCE_CONFIG, ap_info["ssid"],
+                                    ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result)
+
+    PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF")
+    PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"})
+    assert Attenuator.set_att(att_port, 0) is True
+
+    if not test_utility.wait_ap_power_on():
+        Utility.console_log("[{}] failed to power on, skip testing this AP"
+                            .format(ap_info["ssid"]), color="red")
+
+    # 4. run test for TCP Tx, Rx and UDP Tx, Rx
+    for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE):
+        test_utility.run_all_cases(0)
+
+    # 5. log performance and compare with pass standard
+    for throughput_type in test_result:
+        IDF.log_performance("{}_throughput".format(throughput_type),
+                            "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput()))
+    # do check after logging, otherwise test will exit immediately if check fail, some performance can't be logged.
+    for throughput_type in test_result:
+        IDF.check_performance("{}_throughput".format(throughput_type),
+                              test_result[throughput_type].get_best_throughput())
+
+    env.close_dut("iperf")
+
+
+if __name__ == '__main__':
+    test_wifi_throughput_basic(env_config_file="EnvConfig.yml")
+    test_wifi_throughput_with_different_configs(env_config_file="EnvConfig.yml")
+    test_wifi_throughput_vs_rssi(env_config_file="EnvConfig.yml")

examples/wifi/iperf/sdkconfig.defaults

@@ -2,6 +2,8 @@ CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
 CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ=240
 CONFIG_MEMMAP_SMP=y
 
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
 CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
 CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
 CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64

examples/wifi/iperf/sdkconfig.defaults.00

@@ -0,0 +1,6 @@
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=

examples/wifi/iperf/sdkconfig.defaults.01

@@ -0,0 +1,25 @@
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=12
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=48
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=48
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=12
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=12
+
+CONFIG_TCP_SND_BUF_DEFAULT=11488
+CONFIG_TCP_WND_DEFAULT=11488
+CONFIG_TCP_RECVMBOX_SIZE=12
+CONFIG_UDP_RECVMBOX_SIZE=12
+CONFIG_TCPIP_RECVMBOX_SIZE=48
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+

examples/wifi/iperf/sdkconfig.defaults.02

@@ -0,0 +1,25 @@
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=32
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=32
+
+CONFIG_TCP_SND_BUF_DEFAULT=11488
+CONFIG_TCP_WND_DEFAULT=11488
+CONFIG_TCP_RECVMBOX_SIZE=12
+CONFIG_UDP_RECVMBOX_SIZE=12
+CONFIG_TCPIP_RECVMBOX_SIZE=48
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+

examples/wifi/iperf/sdkconfig.defaults.03

@@ -0,0 +1,25 @@
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=32
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=32
+
+CONFIG_TCP_SND_BUF_DEFAULT=32768
+CONFIG_TCP_WND_DEFAULT=32768
+CONFIG_TCP_RECVMBOX_SIZE=64
+CONFIG_UDP_RECVMBOX_SIZE=64
+CONFIG_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+

examples/wifi/iperf/sdkconfig.defaults.04

@@ -0,0 +1,25 @@
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=32
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=32
+
+CONFIG_TCP_SND_BUF_DEFAULT=65535
+CONFIG_TCP_WND_DEFAULT=65535
+CONFIG_TCP_RECVMBOX_SIZE=64
+CONFIG_UDP_RECVMBOX_SIZE=64
+CONFIG_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+

examples/wifi/iperf/sdkconfig.defaults.05

@@ -0,0 +1,29 @@
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=32
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=32
+
+CONFIG_TCP_SND_BUF_DEFAULT=65535
+CONFIG_TCP_WND_DEFAULT=65535
+CONFIG_TCP_RECVMBOX_SIZE=64
+CONFIG_UDP_RECVMBOX_SIZE=64
+CONFIG_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+
+CONFIG_FLASHMODE_QIO=y
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_LWIP_IRAM_OPTIMIZATION=y
+

examples/wifi/iperf/sdkconfig.defaults.06

@@ -0,0 +1,26 @@
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=32
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=32
+
+CONFIG_FREERTOS_UNICORE=y
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_TCP_SND_BUF_DEFAULT=65535
+CONFIG_TCP_WND_DEFAULT=65535
+CONFIG_TCP_RECVMBOX_SIZE=64
+CONFIG_UDP_RECVMBOX_SIZE=64
+CONFIG_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+
+CONFIG_FLASHMODE_QIO=y
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_LWIP_IRAM_OPTIMIZATION=y

examples/wifi/iperf/sdkconfig.defaults.07

@@ -0,0 +1,30 @@
+CONFIG_ESP32_DEFAULT_CPU_FREQ_80=y
+CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ=80
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=32
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=32
+
+CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_TCP_SND_BUF_DEFAULT=65535
+CONFIG_TCP_WND_DEFAULT=65535
+CONFIG_TCP_RECVMBOX_SIZE=64
+CONFIG_UDP_RECVMBOX_SIZE=64
+CONFIG_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+
+CONFIG_FLASHMODE_QIO=y
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_LWIP_IRAM_OPTIMIZATION=y

examples/wifi/iperf/sdkconfig.defaults.99

@@ -0,0 +1,30 @@
+CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ=240
+CONFIG_MEMMAP_SMP=y
+
+CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=4096
+
+CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
+CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=64
+CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y
+CONFIG_ESP32_WIFI_TX_BA_WIN=32
+CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
+CONFIG_ESP32_WIFI_RX_BA_WIN=32
+
+CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_HZ=1000
+
+CONFIG_INT_WDT=
+CONFIG_TASK_WDT=
+
+CONFIG_TCP_SND_BUF_DEFAULT=65535
+CONFIG_TCP_WND_DEFAULT=65535
+CONFIG_TCP_RECVMBOX_SIZE=64
+CONFIG_UDP_RECVMBOX_SIZE=64
+CONFIG_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+
+CONFIG_FLASHMODE_QIO=y
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_LWIP_IRAM_OPTIMIZATION=y

examples/wifi/iperf/test_report.py

@@ -0,0 +1,245 @@
+"""
+this module generates markdown format test report for throughput test.
+
+The test report contains 2 parts:
+
+1. throughput with different configs
+2. throughput with RSSI
+"""
+import os
+
+
+class ThroughputForConfigsReport(object):
+    THROUGHPUT_TYPES = ["tcp_tx", "tcp_rx", "udp_tx", "udp_rx"]
+
+    REPORT_FILE_NAME = "ThroughputForConfigs.md"
+
+    def __init__(self, output_path, ap_ssid, throughput_results, sdkconfig_files):
+        """
+        :param ap_ssid: the ap we expected to use
+        :param throughput_results: config with the following type::
+            {
+                "config_name": {
+                    "tcp_tx": result,
+                    "tcp_rx": result,
+                    "udp_tx": result,
+                    "udp_rx": result,
+                },
+                "config_name2": {},
+            }
+        """
+        self.output_path = output_path
+        self.ap_ssid = ap_ssid
+        self.results = throughput_results
+        self.sdkconfigs = dict()
+        for config_name in sdkconfig_files:
+            self.sdkconfigs[config_name] = self._parse_config_file(sdkconfig_files[config_name])
+        if not os.path.exists(output_path):
+            os.makedirs(output_path)
+        self.sort_order = self.sdkconfigs.keys()
+        self.sort_order.sort()
+
+    @staticmethod
+    def _parse_config_file(config_file_path):
+        sdkconfig = {}
+        with open(config_file_path, "r") as f:
+            for line in f:
+                if not line.isspace():
+                    if line[0] == "#":
+                        continue
+                    name, value = line.split("=")
+                    value = value.strip("\r\n")
+                    sdkconfig[name] = value if value else "n"
+        return sdkconfig
+
+    def _generate_the_difference_between_configs(self):
+        """
+        generate markdown list for different configs::
+
+            default: esp-idf default
+            low:
+                * `config name 1`: old value -> new value
+                * `config name 2`: old value -> new value
+                * ...
+            ...
+
+        """
+
+        data = "## Config Definition:\r\n\r\n"
+
+        def find_difference(base, new):
+            _difference = {}
+            all_configs = set(base.keys())
+            all_configs.update(set(new.keys()))
+            for _config in all_configs:
+                try:
+                    _base_value = base[_config]
+                except KeyError:
+                    _base_value = "null"
+                try:
+                    _new_value = new[_config]
+                except KeyError:
+                    _new_value = "null"
+                if _base_value != _new_value:
+                    _difference[_config] = "{} -> {}".format(_base_value, _new_value)
+            return _difference
+
+        for i, _config_name in enumerate(self.sort_order):
+            current_config = self.sdkconfigs[_config_name]
+
+            if i > 0:
+                previous_config_name = self.sort_order[i-1]
+                previous_config = self.sdkconfigs[previous_config_name]
+            else:
+                previous_config = previous_config_name = None
+
+            if previous_config:
+                # log the difference
+                difference = find_difference(previous_config, current_config)
+                data += "* {} (compared to {}):\r\n".format(_config_name, previous_config_name)
+                for diff_name in difference:
+                    data += "    * `{}`: {}\r\n".format(diff_name, difference[diff_name])
+        return data
+
+    def _generate_report_for_one_type(self, throughput_type):
+        """
+        generate markdown table with the following format::
+
+            | config name | throughput (Mbps) | free heap size (bytes) |
+            |-------------|-------------------|------------------------|
+            | default     | 32.11             | 147500                 |
+            | low         | 32.11             | 147000                 |
+            | medium      | 33.22             | 120000                 |
+            | high        | 43.11             | 100000                 |
+            | max         | 45.22             | 79000                  |
+        """
+        empty = True
+
+        ret = "\r\n### {} {}\r\n\r\n".format(*throughput_type.split("_"))
+        ret += "| config name | throughput (Mbps) | free heap size (bytes) |\r\n"
+        ret += "|-------------|-------------------|------------------------|\r\n"
+        for config in self.sort_order:
+            try:
+                result = self.results[config][throughput_type]
+                throughput = "{:.02f}".format(max(result.throughput_by_att[self.ap_ssid].values()))
+                heap_size = str(result.heap_size)
+                # although markdown table will do alignment
+                # do align here for better text editor presentation
+                ret += "| {:<12}| {:<18}| {:<23}|\r\n".format(config, throughput, heap_size)
+                empty = False
+            except KeyError:
+                pass
+        return ret if not empty else ""
+
+    def generate_report(self):
+        data = "# Throughput for different configs\r\n"
+        data += "\r\nAP: {}\r\n".format(self.ap_ssid)
+
+        for throughput_type in self.THROUGHPUT_TYPES:
+            data += self._generate_report_for_one_type(throughput_type)
+            data += "\r\n------\r\n"
+
+        data += self._generate_the_difference_between_configs()
+
+        with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), "w") as f:
+            f.write(data)
+
+
+class ThroughputVsRssiReport(object):
+
+    REPORT_FILE_NAME = "ThroughputVsRssi.md"
+
+    def __init__(self, output_path, throughput_results):
+        """
+        :param throughput_results: config with the following type::
+            {
+                "tcp_tx": result,
+                "tcp_rx": result,
+                "udp_tx": result,
+                "udp_rx": result,
+            }
+        """
+        self.output_path = output_path
+        self.raw_data_path = os.path.join(output_path, "raw_data")
+        self.results = throughput_results
+        self.throughput_types = self.results.keys()
+        self.throughput_types.sort()
+        if not os.path.exists(self.raw_data_path):
+            os.makedirs(self.raw_data_path)
+
+    def _generate_summary(self):
+        """
+        generate summary with the following format::
+
+            | item    | curve analysis | max throughput (Mbps) |
+            |---------|----------------|-----------------------|
+            | tcp tx  | Success        | 32.11                 |
+            | tcp rx  | Success        | 32.11                 |
+            | udp tx  | Success        | 45.22                 |
+            | udp rx  | Failed         | 55.44                 |
+        """
+
+        ret = "\r\n### Summary\r\n\r\n"
+        ret += "| item    | curve analysis | max throughput (Mbps) |\r\n"
+        ret += "|---------|----------------|-----------------------|\r\n"
+
+        for _type in self.throughput_types:
+            result = self.results[_type]
+            max_throughput = 0.0
+            curve_analysis = "Failed" if result.error_list else "Success"
+            for ap_ssid in result.throughput_by_att:
+                _max_for_ap = max(result.throughput_by_rssi[ap_ssid].values())
+                if _max_for_ap > max_throughput:
+                    max_throughput = _max_for_ap
+            max_throughput = "{:.02f}".format(max_throughput)
+            ret += "| {:<8}| {:<15}| {:<22}|\r\n".format("{}_{}".format(result.proto, result.direction),
+                                                         curve_analysis, max_throughput)
+        return ret
+
+    def _generate_report_for_one_type(self, result):
+        """
+        generate markdown table with the following format::
+
+            ### tcp rx
+
+            Errors:
+
+            * detected error 1
+            * ...
+
+            AP: ap_ssid
+
+            ![throughput Vs RSSI](path to figure)
+
+            AP: ap_ssid
+
+            ![throughput Vs RSSI](path to figure)
+
+        """
+        result.post_analysis()
+        ret = "\r\n### {} {}\r\n".format(result.proto, result.direction)
+        if result.error_list:
+            ret += "\r\nErrors:\r\n\r\n"
+            for error in result.error_list:
+                ret += "* " + error + "\r\n"
+        for ap_ssid in result.throughput_by_rssi:
+            ret += "\r\nAP: {}\r\n".format(ap_ssid)
+            # draw figure
+            file_name = result.draw_throughput_figure(self.raw_data_path, ap_ssid, "rssi")
+            result.draw_throughput_figure(self.raw_data_path, ap_ssid, "att")
+            result.draw_rssi_vs_att_figure(self.raw_data_path, ap_ssid)
+
+            ret += "\r\n![throughput Vs RSSI]({})\r\n".format(os.path.join("raw_data", file_name))
+
+        return ret
+
+    def generate_report(self):
+        data = "# Throughput Vs RSSI\r\n"
+
+        data += self._generate_summary()
+
+        for _type in self.throughput_types:
+            data += self._generate_report_for_one_type(self.results[_type])
+
+        with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), "w") as f:
+            f.write(data)