Merge branch 'feature/ci_flasher_args_json_encrypted_flag' into 'master'

ci: use "encrypted" information in flasher_args.json

Closes IDF-2231

See merge request espressif/esp-idf!11551

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -30,20 +30,55 @@ except ImportError:
     gitlab_api = None
 
 
-def parse_flash_settings(path):
+def parse_encrypted_flag(args, offs, binary):
+    # Find partition entries (e.g. the entries with an offset and a file)
+    for _, entry in args:
+        # If the current entry is a partition, we have to check whether it is
+        # the one we are looking for or not
+        try:
+            if (entry["offset"], entry["file"]) == (offs, binary):
+                return entry["encrypted"] == "true"
+        except (TypeError, KeyError):
+            # TypeError occurs if the entry is a list, which is possible in JSON
+            # data structure.
+            # KeyError occurs if the entry doesn't have "encrypted" field.
+            continue
+
+    # The entry was not found, return None. The caller will have to check
+    # CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT macro
+    return None
+
+
+def parse_flash_settings(path, default_encryption=False):
     file_name = os.path.basename(path)
+
+    # For compatibility reasons, this list contains all the files to be
+    # flashed
+    flash_files = []
+    # The following list only contains the files that need encryption
+    encrypt_files = []
+
     if file_name == "flasher_args.json":
         # CMake version using build metadata file
         with open(path, "r") as f:
             args = json.load(f)
-        flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""]
+
+        for (offs, binary) in args["flash_files"].items():
+            if offs:
+                flash_files.append((offs, binary))
+                encrypted = parse_encrypted_flag(args, offs, binary)
+
+                # default_encryption should be taken into account if and only if
+                # encrypted flag is not provided in the JSON file.
+                if (encrypted is None and default_encryption) or encrypted:
+                    encrypt_files.append((offs, binary))
+
         flash_settings = args["flash_settings"]
         app_name = os.path.splitext(args["app"]["file"])[0]
     else:
         # GNU Make version uses download.config arguments file
         with open(path, "r") as f:
             args = f.readlines()[-1].split(" ")
-            flash_files = []
             flash_settings = {}
             for idx in range(0, len(args), 2):  # process arguments in pairs
                 if args[idx].startswith("--"):
@@ -52,6 +87,9 @@ def parse_flash_settings(path):
                 else:
                     # offs, filename
                     flash_files.append((args[idx], args[idx + 1]))
+            # Parameter default_encryption tells us if the files need encryption
+            if default_encryption:
+                encrypt_files = flash_files
             # we can only guess app name in download.config.
             for p in flash_files:
                 if not os.path.dirname(p[1]) and "partition" not in p[1]:
@@ -60,7 +98,7 @@ def parse_flash_settings(path):
                     break
             else:
                 app_name = None
-    return flash_files, flash_settings, app_name
+    return flash_files, encrypt_files, flash_settings, app_name
 
 
 class Artifacts(object):
@@ -110,8 +148,9 @@ class Artifacts(object):
         return flash_arg_file
 
     def _download_binary_files(self, base_path, job_id, flash_arg_file):
-        flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path,
-                                                                                  flash_arg_file))
+        # Let's ignore the second value returned (encrypt_files) as these
+        # files also appear in the first list
+        flash_files, _, _, app_name = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file))
         artifact_files = [os.path.join(base_path, p[1]) for p in flash_files]
         artifact_files.append(os.path.join(base_path, app_name + ".elf"))
 
@@ -198,7 +237,9 @@ class IDFApp(App.BaseApp):
                                   self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
                 raise AssertionError(msg)
 
-        self.flash_files, self.flash_settings = self._parse_flash_download_config()
+        # In order to keep backward compatibility, flash_files is unchanged.
+        # However, we now have a new attribute encrypt_files.
+        self.flash_files, self.encrypt_files, self.flash_settings = self._parse_flash_download_config()
         self.partition_table = self._parse_partition_table()
 
     def __str__(self):
@@ -275,6 +316,11 @@ class IDFApp(App.BaseApp):
                 ret = os.path.join(self.binary_path, fn)
         return ret
 
+    def _int_offs_abs_paths(self, files_list):
+        return [(int(offs, 0),
+                 os.path.join(self.binary_path, file_path.strip()))
+                for (offs, file_path) in files_list]
+
     def _parse_flash_download_config(self):
         """
         Parse flash download config from build metadata files
@@ -293,16 +339,20 @@ class IDFApp(App.BaseApp):
             # GNU Make version uses download.config arguments file
             path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
 
-        flash_files, flash_settings, app_name = parse_flash_settings(path)
-        # The build metadata file does not currently have details, which files should be encrypted and which not.
-        # Assume that all files should be encrypted if flash encryption is enabled in development mode.
+        # If the JSON doesn't find the encrypted flag for our files, provide
+        # a default encrpytion flag: the macro
+        # CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
         sdkconfig_dict = self.get_sdkconfig()
-        flash_settings["encrypt"] = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
+        default_encryption = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
+
+        flash_files, encrypt_files, flash_settings, _ = parse_flash_settings(path, default_encryption)
 
-        # make file offsets into integers, make paths absolute
-        flash_files = [(int(offs, 0), os.path.join(self.binary_path, file_path.strip())) for (offs, file_path) in flash_files]
+        # Flash setting "encrypt" only and only if all the files to flash
+        # must be encrypted. Else, this parameter should be False.
+        # All files must be encrypted is both file lists are the same
+        flash_settings["encrypt"] = sorted(flash_files) == sorted(encrypt_files)
 
-        return flash_files, flash_settings
+        return self._int_offs_abs_paths(flash_files), self._int_offs_abs_paths(encrypt_files), self.flash_settings
 
     def _parse_partition_table(self):
         """

tools/ci/python_packages/ttfw_idf/IDFDUT.py

@@ -216,9 +216,28 @@ class IDFDUT(DUT.SerialDUT):
         Structured this way so @_uses_esptool will reconnect each time
         """
         flash_files = []
+        encrypt_files = []
         try:
-            # note: opening here prevents us from having to seek back to 0 each time
-            flash_files = [(offs, open(path, "rb")) for (offs, path) in self.app.flash_files]
+            # Open the files here to prevents us from having to seek back to 0
+            # each time. Before opening them, we have to organize the lists the
+            # way esptool.write_flash needs:
+            # If encrypt is provided, flash_files contains all the files to
+            # flash.
+            # Else, flash_files contains the files to be flashed as plain text
+            # and encrypt_files contains the ones to flash encrypted.
+            flash_files = self.app.flash_files
+            encrypt_files = self.app.encrypt_files
+            encrypt = self.app.flash_settings.get("encrypt", False)
+            if encrypt:
+                flash_files = encrypt_files
+                encrypt_files = []
+            else:
+                flash_files = [entry
+                               for entry in flash_files
+                               if entry not in encrypt_files]
+
+            flash_files = [(offs, open(path, "rb")) for (offs, path) in flash_files]
+            encrypt_files = [(offs, open(path, "rb")) for (offs, path) in encrypt_files]
 
             if erase_nvs:
                 address = self.app.partition_table["nvs"]["offset"]
@@ -228,7 +247,18 @@ class IDFDUT(DUT.SerialDUT):
                 nvs_file.seek(0)
                 if not isinstance(address, int):
                     address = int(address, 0)
-                flash_files.append((address, nvs_file))
+                # We have to check whether this file needs to be added to
+                # flash_files list or encrypt_files.
+                # Get the CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT macro
+                # value. If it is set to True, then NVS is always encrypted.
+                sdkconfig_dict = self.app.get_sdkconfig()
+                macro_encryption = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
+                # If the macro is not enabled (plain text flash) or all files
+                # must be encrypted, add NVS to flash_files.
+                if not macro_encryption or encrypt:
+                    flash_files.append((address, nvs_file))
+                else:
+                    encrypt_files.append((address, nvs_file))
 
             # fake flasher args object, this is a hack until
             # esptool Python API is improved
@@ -237,15 +267,18 @@ class IDFDUT(DUT.SerialDUT):
                     for key, value in attributes.items():
                         self.__setattr__(key, value)
 
+            # write_flash expects the parameter encrypt_files to be None and not
+            # an empty list, so perform the check here
             flash_args = FlashArgs({
                 'flash_size': self.app.flash_settings["flash_size"],
                 'flash_mode': self.app.flash_settings["flash_mode"],
                 'flash_freq': self.app.flash_settings["flash_freq"],
                 'addr_filename': flash_files,
+                'encrypt_files': encrypt_files or None,
                 'no_stub': False,
                 'compress': True,
                 'verify': False,
-                'encrypt': self.app.flash_settings.get("encrypt", False),
+                'encrypt': encrypt,
                 'erase_all': False,
             })
 
@@ -255,6 +288,8 @@ class IDFDUT(DUT.SerialDUT):
         finally:
             for (_, f) in flash_files:
                 f.close()
+            for (_, f) in encrypt_files:
+                f.close()
 
     def start_app(self, erase_nvs=ERASE_NVS):
         """