Merge branch 'bugfix/py3_ci' into 'master'

Fix Python3 compatibility issues

See merge request idf/esp-idf!3487

docs/conf_common.py

@@ -27,6 +27,8 @@ import shlex
 
 from local_util import run_cmd_get_output, copy_if_modified
 
+# build_docs on the CI server sometimes fails under Python3. This is a workaround:
+sys.setrecursionlimit(3500)
 
 try:
     builddir = os.environ['BUILDDIR']

docs/docs_common.mk

@@ -17,7 +17,7 @@
 SPHINXOPTS    =
 # note: this is changed from sphinx-build so it depends on default python interpreter, not on /bin/sphinx-build
 # (which will be the most recently installed version of sphinx and may not match)
-SPHINXBUILD   = python2 -m sphinx
+SPHINXBUILD   = python -m sphinx
 PAPER         =
 BUILDDIR      = _build
 

docs/gen-version-specific-includes.py

@@ -170,9 +170,9 @@ def get_version():
     # No tag, look for a branch
     refs = subprocess.check_output(["git", "for-each-ref", "--points-at", "HEAD", "--format", "%(refname)"])
     print("refs:\n%s" % refs)
-    refs = refs.split("\n")
+    refs = refs.split(b"\n")
     # Note: this looks for branches in 'origin' because GitLab CI doesn't check out a local branch
-    branches = [ r.replace("refs/remotes/origin/","").strip() for r in refs if r.startswith("refs/remotes/origin/") ]
+    branches = [ r.replace(b"refs/remotes/origin/",b"").strip() for r in refs if r.startswith(b"refs/remotes/origin/") ]
     if len(branches) == 0:
         # last resort, return the commit (may happen on Gitlab CI sometimes, unclear why)
         return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip(), "commit", False)

examples/protocols/asio/chat_client/asio_chat_client_test.py

@@ -19,8 +19,8 @@ import IDF
 global g_client_response;
 global g_msg_to_client;
 
-g_client_response = ""
-g_msg_to_client = "   3XYZ"
+g_client_response = b""
+g_msg_to_client = b"   3XYZ"
 
 def get_my_ip():
     s1 = socket(AF_INET, SOCK_DGRAM)
@@ -81,11 +81,12 @@ def test_examples_protocol_asio_chat_client(env, extra_data):
     # 3. send host's IP to the client i.e. the `dut1`
     dut1.write(host_ip)
     # 4. client `dut1` should receive a message
-    dut1.expect(g_msg_to_client[4:]) # Strip out the front 4 bytes of message len (see chat_message protocol)
+    dut1.expect(g_msg_to_client[4:].decode()) # Strip out the front 4 bytes of message len (see chat_message protocol)
     # 5. write test message from `dut1` chat_client to the server
     dut1.write(test_msg)
-    while g_client_response == "":
+    while len(g_client_response) == 0:
         time.sleep(1)
+    g_client_response = g_client_response.decode()
     print(g_client_response)
     # 6. evaluate host_server received this message
     if (g_client_response[4:7] == test_msg):
@@ -93,7 +94,7 @@ def test_examples_protocol_asio_chat_client(env, extra_data):
         pass
     else:
         print("Failure!")
-        raise ValueError('Wrong data received from asi tcp server: {} (expected:{})'.format(g_client_response, test_msg))
+        raise ValueError('Wrong data received from asi tcp server: {} (expected:{})'.format(g_client_response[4:7], test_msg))
     thread1.join()
 
 if __name__ == '__main__':

examples/protocols/asio/chat_server/asio_chat_server_test.py

@@ -27,7 +27,7 @@ def test_examples_protocol_asio_chat_server(env, extra_data):
       3. Test connects to server and sends a test message
       4. Test evaluates received test message from server
     """
-    test_msg="   4ABC\n"
+    test_msg=b"   4ABC\n"
     dut1 = env.get_dut("chat_server", "examples/protocols/asio/chat_server")
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "asio_chat_server.bin")

examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py

@@ -28,7 +28,7 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
       4. Test evaluates received test message from server
       5. Test evaluates received test message on server stdout
     """
-    test_msg="echo message from client to server"
+    test_msg=b"echo message from client to server"
     dut1 = env.get_dut("tcp_echo_server", "examples/protocols/asio/tcp_echo_server")
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "asio_tcp_echo_server.bin")
@@ -53,7 +53,7 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
         print("Failure!")
         raise ValueError('Wrong data received from asi tcp server: {} (expoected:{})'.format(data, test_msg))
     # 5. check the client message appears also on server terminal
-    dut1.expect(test_msg)
+    dut1.expect(test_msg.decode())
 
 
 if __name__ == '__main__':

examples/protocols/asio/udp_echo_server/asio_udp_server_test.py

@@ -28,7 +28,7 @@ def test_examples_protocol_asio_udp_server(env, extra_data):
       4. Test evaluates received test message from server
       5. Test evaluates received test message on server stdout
     """
-    test_msg="echo message from client to server"
+    test_msg=b"echo message from client to server"
     dut1 = env.get_dut("udp_echo_server", "examples/protocols/asio/udp_echo_server")
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "asio_udp_echo_server.bin")
@@ -53,7 +53,7 @@ def test_examples_protocol_asio_udp_server(env, extra_data):
         print("Failure!")
         raise ValueError('Wrong data received from asi udp server: {} (expoected:{})'.format(data, test_msg))
     # 5. check the client message appears also on server terminal
-    dut1.expect(test_msg)
+    dut1.expect(test_msg.decode())
 
 if __name__ == '__main__':
     test_examples_protocol_asio_udp_server()

tools/ci/check_artifacts_expire_time.py

@@ -23,7 +23,7 @@ def check_artifacts_expire_time():
 
     print("expire time for jobs:")
 
-    job_names = config.keys()
+    job_names = list(config.keys())
     job_names.sort()
 
     for job_name in job_names:

tools/tiny-test-fw/Utility/Attenuator.py

@@ -20,6 +20,7 @@ This file provide method to control programmable attenuator.
 
 import time
 import serial
+import codecs
 
 
 def set_att(port, att, att_fix=False):
@@ -51,11 +52,11 @@ def set_att(port, att, att_fix=False):
     cmd_hex = "7e7e10{:02x}{:x}".format(att_t, 0x10 + att_t)
     exp_res_hex = "7e7e20{:02x}00{:x}".format(att_t, 0x20 + att_t)
 
-    cmd = cmd_hex.decode("hex")
-    exp_res = exp_res_hex.decode("hex")
+    cmd = codecs.decode(cmd_hex, "hex")
+    exp_res = codecs.decode(exp_res_hex, "hex")
 
     serial_port.write(cmd)
-    res = ""
+    res = b""
 
     for i in range(5):
         res += serial_port.read(20)

tools/tiny-test-fw/Utility/PowerControl.py

@@ -27,15 +27,15 @@ class Control(object):
     @classmethod
     def apc_telnet_make_choice(cls, telnet, choice):
         """ select a choice """
-        telnet.read_until("Event Log")
-        telnet.read_until(">")
-        telnet.write(choice + "\r\n")
+        telnet.read_until(b"Event Log")
+        telnet.read_until(b">")
+        telnet.write(choice.encode() + b"\r\n")
 
     @classmethod
     def apc_telnet_common_action(cls, telnet, check_str, action):
         """ wait until a pattern and then write a line """
-        telnet.read_until(check_str)
-        telnet.write(action + "\r\n")
+        telnet.read_until(check_str.encode())
+        telnet.write(action.encode() + b"\r\n")
 
     @classmethod
     def control(cls, apc_ip, control_dict):
@@ -83,13 +83,13 @@ class Control(object):
             cls.apc_telnet_make_choice(tn, "\033")
 
         # exit to main menu and logout
-        tn.write("\033\r\n")
-        tn.write("\033\r\n")
-        tn.write("\033\r\n")
-        tn.write("4\r\n")
+        tn.write(b"\033\r\n")
+        tn.write(b"\033\r\n")
+        tn.write(b"\033\r\n")
+        tn.write(b"4\r\n")
 
     @classmethod
     def control_rest(cls, apc_ip, outlet, action):
-        outlet_list = range(1, 9)
+        outlet_list = list(range(1, 9)) # has to be a list if we want to remove from it under Python 3
         outlet_list.remove(outlet)
         cls.control(apc_ip, dict.fromkeys(outlet_list, action))

tools/unit-test-app/tools/CreateSectionTable.py

@@ -6,8 +6,8 @@ class Section(object):
     """
     One Section of section table. contains info about section name, address and raw data
     """
-    SECTION_START_PATTERN = re.compile("Contents of section (.+?):")
-    DATA_PATTERN = re.compile("([0-9a-f]{4,8})")
+    SECTION_START_PATTERN = re.compile(b"Contents of section (.+?):")
+    DATA_PATTERN = re.compile(b"([0-9a-f]{4,8})")
 
     def __init__(self, name, start_address, data):
         self.name = name
@@ -52,7 +52,7 @@ class Section(object):
         start_address = 0
         # first find start line
         for i, line in enumerate(raw_data):
-            if "Contents of section " in line:  # do strcmp first to speed up
+            if b"Contents of section " in line:  # do strcmp first to speed up
                 match = cls.SECTION_START_PATTERN.search(line)
                 if match is not None:
                     name = match.group(1)
@@ -60,11 +60,11 @@ class Section(object):
                     break
         else:
             # do some error handling
-            raw_data = [""]  # add a dummy first data line
+            raw_data = [b""]  # add a dummy first data line
 
         def process_data_line(line_to_process):
             # first remove the ascii part
-            hex_part = line_to_process.split("  ")[0]
+            hex_part = line_to_process.split(b"  ")[0]
             # process rest part
             data_list = cls.DATA_PATTERN.findall(hex_part)
             try:
@@ -74,7 +74,7 @@ class Section(object):
 
             def hex_to_str(hex_data):
                 if len(hex_data) % 2 == 1:
-                    hex_data = "0" + hex_data  # append zero at the beginning
+                    hex_data = b"0" + hex_data  # append zero at the beginning
                 _length = len(hex_data)
                 return "".join([chr(int(hex_data[_i:_i + 2], base=16))
                                 for _i in range(0, _length, 2)])

tools/unit-test-app/tools/UnitTestParser.py

@@ -68,7 +68,7 @@ class Parser(object):
         table = CreateSectionTable.SectionTable("section_table.tmp")
         tags = self.parse_tags(os.path.join(config_output_folder, self.SDKCONFIG_FILE))
         test_cases = []
-        with open("case_address.tmp", "r") as f:
+        with open("case_address.tmp", "rb") as f:
             for line in f:
                 # process symbol table like: "3ffb4310 l     O .dram0.data	00000018 test_desc_33$5010"
                 line = line.split()
@@ -235,7 +235,7 @@ class Parser(object):
         dump parsed test cases to YAML file for test bench input
         :param test_cases: parsed test cases
         """
-        with open(os.path.join(self.idf_path, self.TEST_CASE_FILE), "wb+") as f:
+        with open(os.path.join(self.idf_path, self.TEST_CASE_FILE), "w+") as f:
             yaml.dump({"test cases": test_cases}, f, allow_unicode=True, default_flow_style=False)
 
     def copy_module_def_file(self):
@@ -297,7 +297,7 @@ def test_parser():
     }
     sdkconfig = ["123", "789"]
     tags = parser.parse_tags_internal(sdkconfig, config_dependency, parser.CONFIG_PATTERN)
-    assert tags == ['a', 'd', 'f']
+    assert sorted(tags) == ['a', 'd', 'f'] # sorted is required for older Python3, e.g. 3.4.8
 
 
 def main():