Merge branch 'refactor/remove_dependency_on_future' into 'master'

Remove the dependency on the future package

See merge request espressif/esp-idf!18925

components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py

@@ -7,8 +7,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-from __future__ import division, print_function
-
 import argparse
 import array
 import binascii
@@ -21,7 +19,6 @@ import random
 import struct
 import sys
 import zlib
-from builtins import bytes, int, range
 from io import open
 
 try:

examples/peripherals/i2s/i2s_adc_dac/tools/generate_audio_file.py

@@ -1,11 +1,8 @@
 # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Unlicense OR CC0-1.0
-from __future__ import print_function
-
 import os
 import struct
 import wave
-from builtins import range
 
 try:
     from typing import List

examples/protocols/http_server/advanced_tests/scripts/test.py

@@ -1,18 +1,7 @@
 #!/usr/bin/env python
 #
-# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 
 # Utility for testing the web server. Test cases:
 # Assume the device supports 'n' simultaneous open sockets
@@ -139,7 +128,6 @@ import string
 import sys
 import threading
 import time
-from builtins import object, range, str
 
 try:
     import Utility

examples/protocols/http_server/persistent_sockets/pytest_http_server_persistence.py

@@ -2,14 +2,10 @@
 #
 # SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
-from __future__ import division, print_function, unicode_literals
-
 import logging
 import os
 import random
 import sys
-from builtins import range, str
 
 import pytest
 

examples/protocols/http_server/simple/pytest_http_server_simple.py

@@ -2,9 +2,6 @@
 #
 # SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
-from __future__ import division, print_function, unicode_literals
-
 import logging
 import os
 import random
@@ -13,7 +10,6 @@ import string
 import sys
 import threading
 import time
-from builtins import range
 
 import pytest
 

examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py

@@ -1,10 +1,7 @@
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import ssl
 import sys
-from builtins import str
 from threading import Event, Thread
 
 import paho.mqtt.client as mqtt

examples/protocols/mqtt/ws/mqtt_ws_example_test.py

@@ -1,9 +1,6 @@
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import sys
-from builtins import str
 from threading import Event, Thread
 
 import paho.mqtt.client as mqtt

examples/protocols/mqtt/wss/mqtt_wss_example_test.py

@@ -1,10 +1,7 @@
-from __future__ import unicode_literals
-
 import os
 import re
 import ssl
 import sys
-from builtins import str
 from threading import Event, Thread
 
 import paho.mqtt.client as mqtt

examples/protocols/sockets/tcp_client/example_test.py

@@ -5,14 +5,10 @@
 # CONDITIONS OF ANY KIND, either express or implied.
 
 # -*- coding: utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import socket
 import sys
-from builtins import input
 from threading import Event, Thread
 
 import netifaces

examples/protocols/sockets/udp_client/example_test.py

@@ -5,14 +5,10 @@
 # CONDITIONS OF ANY KIND, either express or implied.
 
 # -*- coding: utf-8 -*-
-
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import socket
 import sys
-from builtins import input
 from threading import Event, Thread
 
 import netifaces

examples/wifi/iperf/iperf_test.py

@@ -19,13 +19,10 @@ The test env Example_ShieldBox do need the following config::
     apc_ip: "192.168.1.88"
     pc_nic: "eth0"
 """
-from __future__ import division, unicode_literals
-
 import os
 import re
 import subprocess
 import time
-from builtins import range, str
 
 import ttfw_idf
 from idf_iperf_test_util import Attenuator, IperfUtility, PowerControl, TestReport

tools/ci/check_copyright_ignore.txt

@@ -1769,7 +1769,6 @@ examples/protocols/http_server/advanced_tests/http_server_advanced_test.py
 examples/protocols/http_server/advanced_tests/main/include/tests.h
 examples/protocols/http_server/advanced_tests/main/main.c
 examples/protocols/http_server/advanced_tests/main/tests.c
-examples/protocols/http_server/advanced_tests/scripts/test.py
 examples/protocols/http_server/captive_portal/example_test.py
 examples/protocols/http_server/captive_portal/main/dns_server.c
 examples/protocols/http_server/captive_portal/main/include/dns_server.h

tools/ci/python_packages/idf_http_server_test/adder.py

@@ -3,12 +3,9 @@
 # SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
-from __future__ import print_function, unicode_literals
-
 import argparse
 import http.client
 import logging
-from builtins import range, str
 
 
 def start_session(ip, port):

tools/ci/python_packages/idf_http_server_test/client.py

@@ -3,13 +3,10 @@
 # SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
-from __future__ import print_function, unicode_literals
-
 import argparse
 import errno
 import http.client
 import logging
-from builtins import str
 
 
 def verbose_print(verbosity, *args):

tools/ci/python_packages/idf_http_server_test/test.py

@@ -118,8 +118,6 @@
 #    - Simple GET on /hello/restart_results (returns the leak results)
 
 
-from __future__ import division, print_function
-
 import argparse
 import http.client
 import logging
@@ -129,7 +127,6 @@ import string
 import sys
 import threading
 import time
-from builtins import object, range, str
 
 _verbose_ = False
 

tools/ci/python_packages/idf_iperf_test_util/IperfUtility.py

@@ -4,7 +4,6 @@ import os
 import re
 import subprocess
 import time
-from builtins import object, range, str
 
 from idf_iperf_test_util import LineChart
 from tiny_test_fw import DUT, Utility

tools/gdb_panic_server.py

@@ -25,7 +25,6 @@ import binascii
 import logging
 import struct
 import sys
-from builtins import bytes
 from collections import namedtuple
 
 # Used for type annotations only. Silence linter warnings.

tools/gen_esp_err_to_name.py

@@ -3,22 +3,6 @@
 # SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
-from __future__ import print_function, unicode_literals
-
-import sys
-from typing import Any, List, Optional, TextIO
-
-try:
-    from builtins import object, range, str
-except ImportError:
-    # This should not happen because the Python packages are checked before invoking this script. However, here is
-    # some output which should help if we missed something.
-    print('Import has failed probably because of the missing "future" package. Please install all the packages for '
-          'interpreter {} from the requirements.txt file.'.format(sys.executable))
-    # The path to requirements.txt is not provided because this script could be invoked from an IDF project (then the
-    # requirements.txt from the IDF_PATH should be used) or from the documentation project (then the requirements.txt
-    # for the documentation directory should be used).
-    sys.exit(1)
 import argparse
 import collections
 import fnmatch
@@ -27,6 +11,7 @@ import os
 import re
 import textwrap
 from io import open
+from typing import Any, List, Optional, TextIO
 
 # list files here which should not be parsed
 ignore_files: list  = list()

tools/idf_monitor.py

@@ -29,7 +29,6 @@ import subprocess
 import sys
 import threading
 import time
-from builtins import bytes
 from typing import Any, List, Optional, Type, Union
 
 import serial

tools/idf_size.py

@@ -9,8 +9,6 @@
 # SPDX-FileCopyrightText: 2017-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
-from __future__ import division, print_function, unicode_literals
-
 import argparse
 import collections
 import json
@@ -20,9 +18,16 @@ import sys
 from typing import Any, Callable, Collection, Dict, Iterable, List, Optional, TextIO, Tuple, Union
 
 import yaml
-from future.utils import iteritems
 
-Section = Dict[str, Union[str, int]]
+
+class Section(Dict):
+    # define for python type hints
+    size: int
+    address: int
+    name: str
+    sources: List[Dict]
+
+
 SectionDict = Dict[str, Section]
 
 
@@ -726,7 +731,7 @@ def get_summary(path: str, segments: Dict, sections: Dict, target: str,
         if diff_en:
             reference_json_dic = reference.get_json_dic()
             diff_json_dic = collections.OrderedDict([
-                (k, v - reference_json_dic[k]) for k, v in iteritems(current_json_dic)])
+                (k, v - reference_json_dic[k]) for k, v in current_json_dic.items()])
             output = format_json(collections.OrderedDict([('current', current_json_dic),
                                                           ('reference', reference_json_dic),
                                                           ('diff', diff_json_dic),
@@ -878,7 +883,7 @@ class StructureForDetailedSizes(object):
         Key can be either "archive" (for per-archive data) or "file" (for per-file data) in the result.
         """
         result = {}  # type: Dict[str, Dict[str, int]]
-        for _, section in iteritems(sections):
+        for _, section in sections.items():
             for s in section['sources']:
                 if not s[key] in result:
                     result[s[key]] = {}
@@ -962,7 +967,7 @@ def get_detailed_sizes(sections: Dict, key: str, header: str, as_json: bool=Fals
             header_format = _get_header_format(disp_list)
             output = header_format.format(header, *disp_list)
 
-            for key, data_info in iteritems(data):
+            for key, data_info in data.items():
                 if key not in selection:
                     continue
 
@@ -1003,7 +1008,7 @@ def get_detailed_sizes(sections: Dict, key: str, header: str, as_json: bool=Fals
             output += header_format.format('', *f_print)
             output += header_line
 
-            for key, data_info in iteritems(curr):
+            for key, data_info in curr.items():
                 try:
                     v2 = ref[key]
                 except KeyError:
@@ -1060,7 +1065,7 @@ class StructureForArchiveSymbols(object):
         interested_sections = LinkingSections.filter_sections(sections)
 
         result = dict([(t, {}) for t in interested_sections])  # type: Dict[str, Dict[str, int]]
-        for _, section in iteritems(sections):
+        for _, section in sections.items():
             section_name = section['name']
             if section_name not in interested_sections:
                 continue
@@ -1104,14 +1109,14 @@ def get_archive_symbols(sections: Dict, archive: str, as_json: bool=False, secti
             output = format_json(current)
     else:
         def _get_item_pairs(name: str, section: collections.OrderedDict) -> collections.OrderedDict:
-            return collections.OrderedDict([(key.replace(name + '.', ''), val) for key, val in iteritems(section)])
+            return collections.OrderedDict([(key.replace(name + '.', ''), val) for key, val in section.items()])
 
         def _get_max_len(symbols_dict: Dict) -> Tuple[int, int]:
             # the lists have 0 in them because max() doesn't work with empty lists
             names_max_len = 0
             numbers_max_len = 0
-            for t, s in iteritems(symbols_dict):
-                numbers_max_len = max([numbers_max_len, *[len(str(x)) for _, x in iteritems(s)]])
+            for t, s in symbols_dict.items():
+                numbers_max_len = max([numbers_max_len, *[len(str(x)) for _, x in s.items()]])
                 names_max_len = max([names_max_len, *[len(x) for x in _get_item_pairs(t, s)]])
 
             return names_max_len, numbers_max_len
@@ -1119,12 +1124,12 @@ def get_archive_symbols(sections: Dict, archive: str, as_json: bool=False, secti
         def _get_output(section_symbols: Dict) -> str:
             output = ''
             names_max_len, numbers_max_len  = _get_max_len(section_symbols)
-            for t, s in iteritems(section_symbols):
+            for t, s in section_symbols.items():
                 output += '{}Symbols from section: {}{}'.format(os.linesep, t, os.linesep)
                 item_pairs = _get_item_pairs(t, s)
-                for key, val in iteritems(item_pairs):
+                for key, val in item_pairs.items():
                     output += ' '.join([('\t{:<%d} : {:>%d}\n' % (names_max_len,numbers_max_len)).format(key, val)])
-                section_total = sum([val for _, val in iteritems(item_pairs)])
+                section_total = sum([val for _, val in item_pairs.items()])
                 output += 'Section total: {}{}'.format(section_total, os.linesep)
             return output
 
@@ -1147,8 +1152,8 @@ def get_archive_symbols(sections: Dict, archive: str, as_json: bool=False, secti
                                                           '<CURRENT>',
                                                           '<REFERENCE>',
                                                           '<CURRENT> - <REFERENCE>') + os.linesep
-                current_section_total = sum([val for _, val in iteritems(current_item_pairs)])
-                reference_section_total = sum([val for _, val in iteritems(reference_item_pairs)])
+                current_section_total = sum([val for _, val in current_item_pairs.items()])
+                reference_section_total = sum([val for _, val in reference_item_pairs.items()])
                 diff_section_total = current_section_total - reference_section_total
                 all_item_names = sorted(list(frozenset(current_item_pairs.keys()) |
                                              frozenset(reference_item_pairs.keys())))

tools/kconfig_new/confgen.py

@@ -22,7 +22,6 @@ from collections import defaultdict
 
 import gen_kconfig_doc
 import kconfiglib
-from future.utils import iteritems
 
 __version__ = '0.1'
 
@@ -175,21 +174,6 @@ class DeprecatedOptions(object):
                         f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt))
 
 
-def dict_enc_for_env(dic, encoding=sys.getfilesystemencoding() or 'utf-8'):
-    """
-    This function can be deleted after dropping support for Python 2.
-    There is no rule for it that environment variables cannot be Unicode but usually people try to avoid it.
-    The upstream kconfiglib cannot detect strings properly if the environment variables are "unicode". This is problem
-    only in Python 2.
-    """
-    if sys.version_info[0] >= 3:
-        return dic
-    ret = dict()
-    for (key, value) in iteritems(dic):
-        ret[key.encode(encoding)] = value.encode(encoding)
-    return ret
-
-
 def main():
     parser = argparse.ArgumentParser(description='confgen.py v%s - Config Generation Tool' % __version__, prog=os.path.basename(sys.argv[0]))
 
@@ -251,7 +235,7 @@ def main():
 
     if args.env_file is not None:
         env = json.load(args.env_file)
-        os.environ.update(dict_enc_for_env(env))
+        os.environ.update(env)
 
     config = kconfiglib.Kconfig(args.kconfig)
     config.warn_assign_redun = False

tools/kconfig_new/confserver.py

@@ -3,8 +3,6 @@
 # Long-running server process uses stdin & stdout to communicate JSON
 # with a caller
 #
-from __future__ import print_function
-
 import argparse
 import json
 import os
@@ -66,7 +64,7 @@ def main():
 
     if args.env_file is not None:
         env = json.load(args.env_file)
-        os.environ.update(confgen.dict_enc_for_env(env))
+        os.environ.update(env)
 
     run_server(args.kconfig, args.config, args.sdkconfig_rename)
 

tools/kconfig_new/test/confgen/test_confgen.py

@@ -8,8 +8,6 @@ import tempfile
 import textwrap
 import unittest
 
-from future.utils import iteritems
-
 
 class ConfgenBaseTestCase(unittest.TestCase):
     @classmethod
@@ -36,7 +34,7 @@ class ConfgenBaseTestCase(unittest.TestCase):
     def invoke_confgen(self, args):
         call_args = [sys.executable, '../../confgen.py']
 
-        for (k, v) in iteritems(args):
+        for (k, v) in args.items():
             if k != 'output':
                 if isinstance(v, type('')):  # easy Python 2/3 compatible str/unicode
                     call_args += ['--{}'.format(k), v]

tools/ldgen/ldgen.py

@@ -21,14 +21,6 @@ from ldgen.linker_script import LinkerScript
 from ldgen.sdkconfig import SDKConfig
 from pyparsing import ParseException, ParseFatalException
 
-try:
-    import confgen
-except Exception:
-    parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-    kconfig_new_dir = os.path.abspath(parent_dir_name + '/kconfig_new')
-    sys.path.insert(0, kconfig_new_dir)
-    import confgen
-
 
 def _update_environment(args):
     env = [(name, value) for (name,value) in (e.split('=',1) for e in args.env)]
@@ -38,7 +30,7 @@ def _update_environment(args):
 
     if args.env_file is not None:
         env = json.load(args.env_file)
-        os.environ.update(confgen.dict_enc_for_env(env))
+        os.environ.update(env)
 
 
 def main():

tools/mass_mfg/mfg_gen.py

@@ -4,15 +4,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-from __future__ import print_function
-
 import argparse
 import csv
 import distutils.dir_util
 import os
 import sys
-
-from future.moves.itertools import zip_longest
+from itertools import zip_longest
 
 try:
     idf_path = os.environ['IDF_PATH']

tools/mkdfu.py

@@ -21,8 +21,6 @@ import zlib
 from collections import namedtuple
 from functools import partial
 
-from future.utils import iteritems
-
 try:
     import typing
 except ImportError:
@@ -312,9 +310,9 @@ def main():  # type: () -> None
 
         with open(args.json) as f:
             files += [(int(addr, 0),
-                       process_json_file(f_name)) for addr, f_name in iteritems(json.load(f)['flash_files'])]
+                       process_json_file(f_name)) for addr, f_name in json.load(f)['flash_files'].items()]
 
-    files = sorted([(addr, f_name.decode('utf-8') if isinstance(f_name, type(b'')) else f_name) for addr, f_name in iteritems(dict(files))],
+    files = sorted([(addr, f_name) for addr, f_name in dict(files).items()],
                    key=lambda x: x[0])  # remove possible duplicates and sort based on the address
 
     cmd_args = {'output_file': args.output_file,

tools/mkuf2.py

@@ -13,8 +13,6 @@ import struct
 from functools import partial
 from typing import Dict, List
 
-from future.utils import iteritems
-
 
 def round_up_int_div(n: int, d: int) -> int:
     # equivalent to math.ceil(n / d)
@@ -172,15 +170,15 @@ def main() -> None:
                 flash_dic = dict((x['offset'], x['file']) for x in bin_selection)
             except KeyError:
                 print('Invalid binary was selected.')
-                valid = [k if all(x in v for x in ('offset', 'file')) else None for k, v in iteritems(json_content)]
+                valid = [k if all(x in v for x in ('offset', 'file')) else None for k, v in json_content.items()]
                 print('Valid ones:', ' '.join(x for x in valid if x))
                 exit(1)
         else:
             flash_dic = json_content['flash_files']
 
-        files += [(parse_addr(addr), process_json_file(f_name)) for addr, f_name in iteritems(flash_dic)]
+        files += [(parse_addr(addr), process_json_file(f_name)) for addr, f_name in flash_dic.items()]
 
-    files = sorted([(addr, f_name) for addr, f_name in iteritems(dict(files))],
+    files = sorted([(addr, f_name) for addr, f_name in dict(files).items()],
                    key=lambda x: x[0])  # remove possible duplicates and sort based on the address
 
     cmd_args = {'output_file': args.output_file,

tools/requirements/requirements.core.txt

@@ -3,7 +3,6 @@
 setuptools
 click
 pyserial
-future
 cryptography
 pyparsing
 pyelftools

tools/test_idf_monitor/run_test_idf_monitor.py

@@ -3,8 +3,6 @@
 # SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
-from __future__ import print_function, unicode_literals
-
 import errno
 import filecmp
 import os
@@ -16,7 +14,6 @@ import sys
 import tempfile
 import threading
 import time
-from builtins import object
 from io import open
 
 XTENSA_ARGS = '--toolchain-prefix xtensa-esp32-elf-'