Merge branch 'bugfix/gen_crt_py3_utf' into 'master'

Fixes issues encoding issues python3 scripts for unicode chars

See merge request espressif/esp-idf!9094

components/mbedtls/esp_crt_bundle/gen_crt_bundle.py

@@ -30,6 +30,7 @@ import struct
 import argparse
 import csv
 import re
+from io import open
 
 try:
     from cryptography import x509
@@ -80,7 +81,7 @@ class CertificateBundle:
         try:
             if file_path.endswith('.pem'):
                 status("Parsing certificates from %s" % file_path)
-                with open(file_path, 'r') as f:
+                with open(file_path, 'r', encoding='utf-8') as f:
                     crt_str = f.read()
                     self.add_from_pem(crt_str)
                     return True
@@ -153,7 +154,7 @@ class CertificateBundle:
     def add_with_filter(self, crts_path, filter_path):
 
         filter_set = set()
-        with open(filter_path, 'r') as f:
+        with open(filter_path, 'r', encoding='utf-8') as f:
             csv_reader = csv.reader(f, delimiter=',')
 
             # Skip header
@@ -163,7 +164,7 @@ class CertificateBundle:
 
         status("Parsing certificates from %s" % crts_path)
         crt_str = []
-        with open(crts_path, 'r') as f:
+        with open(crts_path, 'r', encoding='utf-8') as f:
             crt_str = f.read()
 
             # Split all certs into a list of (name, certificate string) tuples

components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/non_ascii_crt.pem

@@ -0,0 +1,23 @@
+NetLock Arany (Class Gold) Főtanúsítvány
+========================================
+-----BEGIN CERTIFICATE-----
+MIIEFTCCAv2gAwIBAgIGSUEs5AAQMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYDVQQGEwJIVTERMA8G
+A1UEBwwIQnVkYXBlc3QxFTATBgNVBAoMDE5ldExvY2sgS2Z0LjE3MDUGA1UECwwuVGFuw7pzw610
+dsOhbnlraWFkw7NrIChDZXJ0aWZpY2F0aW9uIFNlcnZpY2VzKTE1MDMGA1UEAwwsTmV0TG9jayBB
+cmFueSAoQ2xhc3MgR29sZCkgRsWRdGFuw7pzw610dsOhbnkwHhcNMDgxMjExMTUwODIxWhcNMjgx
+MjA2MTUwODIxWjCBpzELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRUwEwYDVQQKDAxO
+ZXRMb2NrIEtmdC4xNzA1BgNVBAsMLlRhbsO6c8OtdHbDoW55a2lhZMOzayAoQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcykxNTAzBgNVBAMMLE5ldExvY2sgQXJhbnkgKENsYXNzIEdvbGQpIEbFkXRhbsO6
+c8OtdHbDoW55MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxCRec75LbRTDofTjl5Bu
+0jBFHjzuZ9lk4BqKf8owyoPjIMHj9DrTlF8afFttvzBPhCf2nx9JvMaZCpDyD/V/Q4Q3Y1GLeqVw
+/HpYzY6b7cNGbIRwXdrzAZAj/E4wqX7hJ2Pn7WQ8oLjJM2P+FpD/sLj916jAwJRDC7bVWaaeVtAk
+H3B5r9s5VA1lddkVQZQBr17s9o3x/61k/iCa11zr/qYfCGSji3ZVrR47KGAuhyXoqq8fxmRGILdw
+fzzeSNuWU7c5d+Qa4scWhHaXWy+7GRWF+GmF9ZmnqfI0p6m2pgP8b4Y9VHx2BJtr+UBdADTHLpl1
+neWIA6pN+APSQnbAGwIDAKiLo0UwQzASBgNVHRMBAf8ECDAGAQH/AgEEMA4GA1UdDwEB/wQEAwIB
+BjAdBgNVHQ4EFgQUzPpnk/C2uNClwB7zU/2MU9+D15YwDQYJKoZIhvcNAQELBQADggEBAKt/7hwW
+qZw8UQCgwBEIBaeZ5m8BiFRhbvG5GK1Krf6BQCOUL/t1fC8oS2IkgYIL9WHxHG64YTjrgfpioTta
+YtOUZcTh5m2C+C8lcLIhJsFyUR+MLMOEkMNaj7rP9KdlpeuY0fsFskZ1FSNqb4VjMIDw1Z4fKRzC
+bLBQWV2QWzuoDTDPv31/zvGdg73JRm4gpvlhUbohL3u+pRVjodSVh/GeufOJ8z2FuLjbvrW5Kfna
+NwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2XjG4Kvte9nHfRCaexOYNkbQu
+dZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E=
+-----END CERTIFICATE-----

components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py

@@ -22,6 +22,7 @@ pem_test_file       = 'entrust.pem'
 verified_der_bundle = 'baltimore_crt_bundle'
 verified_pem_bundle = 'entrust_crt_bundle'
 invalid_test_file   = 'invalid_crt.pem'
+non_ascii_file      = 'non_ascii_crt.pem'
 ca_crts_all_file    = 'cacrt_all.pem'
 
 
@@ -72,6 +73,12 @@ class GenCrtBundleTests(Py23TestCase):
         with self.assertRaisesRegex(gen_crt_bundle.InputError, "No certificate found"):
             bundle.add_from_pem("")
 
+    def test_non_ascii_crt_input(self):
+        bundle = gen_crt_bundle.CertificateBundle()
+
+        bundle.add_from_file(test_crts_path  + non_ascii_file)
+        self.assertTrue(len(bundle.certificates))
+
 
 if __name__ == "__main__":
     unittest.main()

tools/ci/check_public_headers.py

@@ -28,6 +28,7 @@ import argparse
 import queue
 from threading import Thread, Event
 import tempfile
+from io import open
 
 
 class HeaderFailed(Exception):
@@ -60,7 +61,7 @@ class HeaderFailedContainsCode(HeaderFailed):
 def exec_cmd_to_temp_file(what, suffix=""):
     out_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
     rc, out, err = exec_cmd(what, out_file)
-    with open(out_file.name, "r") as f:
+    with open(out_file.name, "r", encoding='utf-8') as f:
         out = f.read()
     return rc, out, err, out_file.name
 
@@ -129,6 +130,11 @@ class PublicHeaderChecker:
                         self.check_one_header(task, num)
                     except HeaderFailed as e:
                         self.failed_queue.put("{}: Failed! {}".format(task, e))
+                    except Exception as e:
+                        # Makes sure any unexpected exceptions causes the program to terminate
+                        self.failed_queue.put("{}: Failed! {}".format(task, e))
+                        self.terminate.set()
+                        raise
 
     def get_failed(self):
         return list(self.failed_queue.queue)
@@ -166,6 +172,7 @@ class PublicHeaderChecker:
             raise HeaderFailedCppGuardMissing()
         else:
             self.compile_one_header(header)
+            temp_header = None
             try:
                 _, _, _, temp_header = exec_cmd_to_temp_file(["sed", "/#include/d; /#error/d", header], suffix=".h")
                 res = self.preprocess_one_header(temp_header, num, ignore_sdkconfig_issue=True)
@@ -174,7 +181,8 @@ class PublicHeaderChecker:
                 elif res == self.PREPROC_OUT_DIFFERENT_NO_EXT_C_HDR_FAILED:
                     raise HeaderFailedCppGuardMissing()
             finally:
-                os.unlink(temp_header)
+                if temp_header:
+                    os.unlink(temp_header)
 
     def compile_one_header(self, header):
         rc, out, err = exec_cmd([self.gcc, "-S", "-o-", "-include", header, self.main_c] + self.include_dir_flags)
@@ -238,7 +246,7 @@ class PublicHeaderChecker:
         project_dir = os.path.join(idf_path, "examples", "get-started", "blink")
         subprocess.check_call(["idf.py", "reconfigure"], cwd=project_dir)
         build_commands_json = os.path.join(project_dir, "build", "compile_commands.json")
-        with open(build_commands_json, "r") as f:
+        with open(build_commands_json, "r", encoding='utf-8') as f:
             build_command = json.load(f)[0]["command"].split()
         include_dir_flags = []
         include_dirs = []
@@ -303,7 +311,7 @@ def check_all_headers():
 
     # process excluded files and dirs
     exclude_file = os.path.join(os.path.dirname(__file__), args.exclude_file)
-    with open(exclude_file, "r") as f:
+    with open(exclude_file, "r", encoding='utf-8') as f:
         lines = [line.rstrip() for line in f]
     ignore_files = []
     ignore_dirs = []

tools/find_build_apps/common.py

@@ -8,6 +8,7 @@ from collections import namedtuple
 import logging
 import json
 import typing
+from io import open
 
 DEFAULT_TARGET = "esp32"
 
@@ -320,7 +321,7 @@ class BuildSystem(object):
             readme_path = get_md_or_rst(os.path.dirname(app_path))
         if not readme_path:
             return None
-        with open(readme_path, "r") as readme_file:
+        with open(readme_path, "r", encoding='utf8') as readme_file:
             return readme_file.read()
 
     @staticmethod