Merge branch 'update/mfg_util' into 'master'

mfg_util: Add changes to mfg_util as per changes in nvs_util

See merge request idf/esp-idf!3573

.flake8

@@ -159,5 +159,3 @@ exclude =
         components/wifi_provisioning/python/wifi_config_pb2.py,
         components/wifi_provisioning/python/wifi_constants_pb2.py,
         examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
-    # temporary list (should be empty)
-        tools/mass_mfg/mfg_gen.py,

components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py

@@ -501,7 +501,7 @@ class NVS(object):
     def create_new_page(self, is_rsrv_page=False):
         # Update available size as each page is created
         if self.size == 0:
-            raise InsufficientSizeError("Size parameter is is less than the size of data in csv.Please increase size.")
+            raise InsufficientSizeError("Size parameter is less than the size of data in csv.Please increase size.")
         if not is_rsrv_page:
             self.size = self.size - Page.PAGE_PARAMS["max_size"]
         self.page_num += 1

components/nvs_flash/test_nvs_host/test_nvs.cpp

@@ -2166,6 +2166,7 @@ TEST_CASE("check and read data from partition generated via partition generation
 
     check_nvs_part_gen_args("test", 3, "../nvs_partition_generator/testdata/sample_singlepage_blob.bin", false, NULL);
 
+    childpid = fork();
     if (childpid == 0) {
         exit(execlp("rm", " rm",
                     "-rf",
@@ -2216,6 +2217,7 @@ TEST_CASE("check and read data from partition generated via partition generation
 
     check_nvs_part_gen_args("test", 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin",false,NULL);
 
+    childpid = fork();
     if (childpid == 0) {
         exit(execlp("rm", " rm",
                     "-rf",
@@ -2228,6 +2230,177 @@ TEST_CASE("check and read data from partition generated via partition generation
     }
 }
 
+TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support disabled", "[mfg_gen]")
+{
+    int childpid = fork();
+    int status;
+
+    if (childpid == 0) {
+        exit(execlp("bash", "bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \
+                    cp -rf ../nvs_partition_generator/testdata . | \
+                    mkdir -p ../../../tools/mass_mfg/host_test",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+        childpid = fork();
+        if (childpid == 0) {
+            exit(execlp("python", "python",
+                        "../../../tools/mass_mfg/mfg_gen.py",
+                        "--conf",
+                        "../../../tools/mass_mfg/samples/sample_config.csv",
+                        "--values",
+                        "../../../tools/mass_mfg/samples/sample_values_singlepage_blob.csv",
+                        "--prefix",
+                        "Test",
+                        "--size",
+                        "0x3000",
+                        "--outdir",
+                        "../../../tools/mass_mfg/host_test",
+                        "--version",
+                        "v1",NULL));
+
+        } else {
+            CHECK(childpid > 0);
+            waitpid(childpid, &status, 0);
+            CHECK(WEXITSTATUS(status) != -1);
+
+            childpid = fork();
+            if (childpid == 0) {
+                exit(execlp("python", "python",
+                            "../nvs_partition_generator/nvs_partition_gen.py",
+                            "--input",
+                            "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
+                            "--output",
+                            "../nvs_partition_generator/Test-1-partition.bin",
+                            "--size",
+                            "0x3000",
+                            "--version",
+                            "v1",NULL));
+
+            } else {
+                CHECK(childpid > 0);
+                waitpid(childpid, &status, 0);
+                CHECK(WEXITSTATUS(status) != -1);
+
+            }
+
+        }
+
+    }
+
+    SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
+    check_nvs_part_gen_args("test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL);
+
+    SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin");
+    check_nvs_part_gen_args("test", 3, "testdata/sample_singlepage_blob.bin", false, NULL);
+
+
+    childpid = fork();
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    rm -rf mfg_testdata | \
+                    rm -rf testdata",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+    }
+
+}
+
+TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support enabled", "[mfg_gen]")
+{
+    int childpid = fork();
+    int status;
+
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \
+                    cp -rf ../nvs_partition_generator/testdata . | \
+                    mkdir -p ../../../tools/mass_mfg/host_test",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+        childpid = fork();
+        if (childpid == 0) {
+            exit(execlp("python", "python",
+                        "../../../tools/mass_mfg/mfg_gen.py",
+                        "--conf",
+                        "../../../tools/mass_mfg/samples/sample_config.csv",
+                        "--values",
+                        "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv",
+                        "--prefix",
+                        "Test",
+                        "--size",
+                        "0x4000",
+                        "--outdir",
+                        "../../../tools/mass_mfg/host_test",
+                        "--version",
+                        "v2",NULL));
+
+        } else {
+            CHECK(childpid > 0);
+            waitpid(childpid, &status, 0);
+            CHECK(WEXITSTATUS(status) != -1);
+
+            childpid = fork();
+            if (childpid == 0) {
+                exit(execlp("python", "python",
+                            "../nvs_partition_generator/nvs_partition_gen.py",
+                            "--input",
+                            "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
+                            "--output",
+                            "../nvs_partition_generator/Test-1-partition.bin",
+                            "--size",
+                            "0x4000",
+                            "--version",
+                            "v2",NULL));
+
+            } else {
+                CHECK(childpid > 0);
+                waitpid(childpid, &status, 0);
+                CHECK(WEXITSTATUS(status) != -1);
+
+            }
+
+        }
+
+    }
+
+    SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
+    check_nvs_part_gen_args("test", 4, "mfg_testdata/sample_multipage_blob.bin", false, NULL);
+
+    SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin");
+    check_nvs_part_gen_args("test", 4, "testdata/sample_multipage_blob.bin", false, NULL);
+
+    childpid = fork();
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    rm -rf mfg_testdata | \
+                    rm -rf testdata",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+    }
+
+}
+
 #if CONFIG_NVS_ENCRYPTION
 TEST_CASE("check underlying xts code for 32-byte size sector encryption", "[nvs]")
 {
@@ -2614,6 +2787,246 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
         }
     }
 
+}
+
+TEST_CASE("check and read data from partition generated via manufacturing utility with encryption enabled using sample keyfile", "[mfg_gen]")
+{
+    int childpid = fork();
+    int status;
+
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \
+                    cp -rf ../nvs_partition_generator/testdata . | \
+                    mkdir -p ../../../tools/mass_mfg/host_test",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+        childpid = fork();
+        if (childpid == 0) {
+            exit(execlp("python", "python",
+                        "../../../tools/mass_mfg/mfg_gen.py",
+                        "--conf",
+                        "../../../tools/mass_mfg/samples/sample_config.csv",
+                        "--values",
+                        "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv",
+                        "--prefix",
+                        "Test",
+                        "--size",
+                        "0x4000",
+                        "--outdir",
+                        "../../../tools/mass_mfg/host_test",
+                        "--version",
+                        "v2",
+                        "--encrypt",
+                        "true",
+                        "--keyfile",
+                        "mfg_testdata/sample_encryption_keys.bin",NULL));
+
+        } else {
+            CHECK(childpid > 0);
+            waitpid(childpid, &status, 0);
+            CHECK(WEXITSTATUS(status) != -1);
+
+            childpid = fork();
+            if (childpid == 0) {
+                exit(execlp("python", "python",
+                            "../nvs_partition_generator/nvs_partition_gen.py",
+                            "--input",
+                            "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
+                            "--output",
+                            "../nvs_partition_generator/Test-1-partition-encrypted.bin",
+                            "--size",
+                            "0x4000",
+                            "--version",
+                            "v2",
+                            "--encrypt",
+                            "true",
+                            "--keyfile",
+                            "testdata/sample_encryption_keys.bin",NULL));
+
+            } else {
+                CHECK(childpid > 0);
+                waitpid(childpid, &status, 0);
+                CHECK(WEXITSTATUS(status) != -1);
+
+            }
+
+        }
+
+    }
+
+    SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
+
+    TEST_ESP_OK(nvs_flash_deinit());
+
+    nvs_sec_cfg_t cfg;
+    for(int count = 0; count < NVS_KEY_SIZE; count++) {
+        cfg.eky[count] = 0x11;
+        cfg.tky[count] = 0x22;
+    }
+
+    check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg);
+
+    SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition-encrypted.bin");
+
+    TEST_ESP_OK(nvs_flash_deinit());
+
+    check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg);
+
+
+    childpid = fork();
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    rm -rf mfg_testdata | \
+                    rm -rf testdata",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+    }
+
+}
+
+TEST_CASE("check and read data from partition generated via manufacturing utility with encryption enabled using new generated key", "[mfg_gen]")
+{
+    int childpid = fork();
+    int status;
+
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \
+                    cp -rf ../nvs_partition_generator/testdata . | \
+                    mkdir -p ../../../tools/mass_mfg/host_test",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+        childpid = fork();
+        if (childpid == 0) {
+            exit(execlp("python", "python",
+                        "../../../tools/mass_mfg/mfg_gen.py",
+                        "--keygen",
+                        "true",
+                        "--outdir",
+                        "../../../tools/mass_mfg/host_test",
+                        "--keyfile",
+                        "encr_keys_host_test.bin",NULL));
+
+        } else {
+            CHECK(childpid > 0);
+            waitpid(childpid, &status, 0);
+            CHECK(WEXITSTATUS(status) != -1);
+
+            childpid = fork();
+            if (childpid == 0) {
+                exit(execlp("python", "python",
+                            "../../../tools/mass_mfg/mfg_gen.py",
+                            "--conf",
+                            "../../../tools/mass_mfg/samples/sample_config.csv",
+                            "--values",
+                            "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv",
+                            "--prefix",
+                            "Test",
+                            "--size",
+                            "0x4000",
+                            "--outdir",
+                            "../../../tools/mass_mfg/host_test",
+                            "--version",
+                            "v2",
+                            "--encrypt",
+                            "true",
+                            "--keyfile",
+                            "../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL));
+
+            } else {
+                CHECK(childpid > 0);
+                waitpid(childpid, &status, 0);
+                CHECK(WEXITSTATUS(status) != -1);
+
+                childpid = fork();
+                if (childpid == 0) {
+                    exit(execlp("python", "python",
+                                "../nvs_partition_generator/nvs_partition_gen.py",
+                                "--input",
+                                "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
+                                "--output",
+                                "../nvs_partition_generator/Test-1-partition-encrypted.bin",
+                                "--size",
+                                "0x4000",
+                                "--version",
+                                "v2",
+                                "--encrypt",
+                                "true",
+                                "--keyfile",
+                                "../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL));
+
+                } else {
+                    CHECK(childpid > 0);
+                    waitpid(childpid, &status, 0);
+                    CHECK(WEXITSTATUS(status) != -1);
+
+                }
+
+            }
+
+        }
+
+    }
+
+
+    SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
+
+    char buffer[64];
+    FILE *fp;
+
+    fp = fopen("../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin","rb");
+    fread(buffer,sizeof(buffer),1,fp);
+
+    fclose(fp);
+
+    TEST_ESP_OK(nvs_flash_deinit());
+
+    nvs_sec_cfg_t cfg;
+
+    for(int count = 0; count < NVS_KEY_SIZE; count++) {
+        cfg.eky[count] = buffer[count] & 255;
+        cfg.tky[count] = buffer[count+32] & 255;
+    }
+
+    check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg);
+
+    SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition-encrypted.bin");
+
+    TEST_ESP_OK(nvs_flash_deinit());
+
+    check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg);
+
+    childpid = fork();
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf keys | \
+                    rm -rf mfg_testdata | \
+                    rm -rf testdata | \
+                    rm -rf ../../../tools/mass_mfg/host_test",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) != -1);
+
+    }
+
 }
 #endif
 

tools/mass_mfg/docs/README.rst

@@ -23,6 +23,8 @@ Prerequisites
 
 .. note:: Make sure the python path is set in the PATH environment variable before using this utility.
 
+Make sure to include packages from `requirement.txt` in top level IDF directory.
+
 Workflow
 -----------
 
@@ -63,7 +65,7 @@ Below is a sample example of such a configuration file::
 
 	app,namespace,
 	firmware_key,data,hex2bin
-	serial_no,data,i32,REPEAT
+	serial_no,data,string,REPEAT
 	device_no,data,i32
 
 
@@ -89,11 +91,11 @@ Each row should have the ``value`` of the corresponding keys, separated by comma
 Below is a sample example of such a values file::
 
 	id,firmware_key,serial_no,device_no 
-	1,1a2b3c4d5e6faabb,111,101 
+	1,1a2b3c4d5e6faabb,A1,101 
 	2,1a2b3c4d5e6fccdd,,102 
 	3,1a2b3c4d5e6feeff,,103 
 
-.. note:: *A new Master CSV Values File is created in the same folder as the input Master CSV File with the values inserted at each line for the key with 'REPEAT' tag.* 
+.. note:: *If 'REPEAT' tag is present, a new Master CSV Values File is created in the same folder as the input Master CSV File with the values inserted at each line for the key with 'REPEAT' tag.* 
 
 .. note:: *Intermediate CSV files are created by this utility which are input to the nvs partition utility to generate the binary files.*
 
@@ -114,14 +116,17 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 *Sample CSV Configuration file and Master CSV Values file is provided with this utility.* 
 
 **Usage**::
+    
+    $ ./mfg_gen.py [-h] [--conf CONFIG_FILE] [--values VALUES_FILE]
+                    [--prefix PREFIX] [--fileid FILEID] [--outdir OUTDIR]
+                    [--size PART_SIZE] [--version {v1,v2}]
+                    [--keygen {true,false}] [--encrypt {true,false}]
+                    [--keyfile KEYFILE]
 
-    $ ./mfg_gen.py [-h] --size PART_SIZE --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
 
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |                                   
 +========================+==============================================================================================+
-| --size PART_SIZE       |  Size of NVS Partition in bytes (must be multiple of 4096)                                   |
-+------------------------+----------------------------------------------------------------------------------------------+
 | --conf CONFIG_FILE     |  the input configuration csv file                                                            |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --values VALUES_FILE   |  the input values csv file                                                                   |
@@ -133,11 +138,45 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 +------------------------+----------------------------------------------------------------------------------------------+
 | --outdir OUTDIR        | the output directory to store the files created (Default: current directory)                 |
 +------------------------+----------------------------------------------------------------------------------------------+
+| --size PART_SIZE       |  Size of NVS Partition in bytes (must be multiple of 4096)                                   |
++------------------------+----------------------------------------------------------------------------------------------+
+| --version {v1,v2}      | Set version. Default: v2                                                                     |
++------------------------+----------------------------------------------------------------------------------------------+
+| --keygen {true,false}  | Generate keys for encryption.                                                                |
+|                        | Default: false                                                                               |
++------------------------+----------------------------------------------------------------------------------------------+
+| --encrypt {true,false} | Set encryption mode. Default: false                                                          |
++------------------------+----------------------------------------------------------------------------------------------+
+| --keyfile KEYFILE      | File having key for encryption (Applicable only if encryption mode is true)                  |
++------------------------+----------------------------------------------------------------------------------------------+
 
-**You can use the below command to run this utility with the sample files provided**::
+*You can use the below commands to run this utility with the sample files provided*::
    
-   $ ./mfg_gen.py --size 0x3000 --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
+    $ ./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_singlepage_blob.csv --prefix Fan --size 0x3000
+
+    $ ./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_multipage_blob.csv --prefix Fan --size 0x4000
+    
+.. note:: When you use this utility to generate per device instance factory images --conf, --values, --prefix and --size arguments are mandatory.
+    
+    $ ./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_singlepage_blob.csv --prefix Fan --size 0x3000 --outdir tmp
+
+.. note:: The --outdir directory is created if not present. 
+
+.. note:: The file path given in the ``file`` type in the values file is expected to be relative to the current directory from which you are running the utility.
+    
+    $ ./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_singlepage_blob.csv --prefix Fan --size 0x3000 --encrypt true --keygen true
+
+.. note:: ``keys/`` directory is generated with the encryption keys filename of the form ``prefix-fileid-keys.bin``.
+    
+*You can also run the below command to use the utility to* **only** *generate encryption keys binary file ( following example 'keys/' directory is created in current path), which can further be used to encrypt per device instance factory images*::
+    
+    $ ./mfg_gen.py --keygen true
+    
+    $ ./mfg_gen.py --keygen true --keyfile encr_keys.bin
+
+.. note:: When running utility to generate only ``keys``, if --keyfile is given it will generate encryption keys with filename given in --keyfile argument.
 
+.. note:: When you use this utility to generate only encryption keys --keygen argument is mandatory.
 
 .. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file.
 

tools/mass_mfg/mfg_gen.py

@@ -22,10 +22,14 @@ import sys
 import os
 import csv
 import argparse
-import shutil
 import distutils.dir_util
-sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
-import nvs_partition_gen
+
+try:
+    sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
+    import nvs_partition_gen
+except Exception as e:
+    print(e)
+    sys.exit("Please check IDF_PATH")
 
 
 def verify_values_exist(input_values_file, keys_in_values_file):
@@ -36,13 +40,13 @@ def verify_values_exist(input_values_file, keys_in_values_file):
 
     values_file = open(input_values_file, 'r')
     values_file_reader = csv.reader(values_file, delimiter=',')
-    keys = next(values_file_reader)
+    next(values_file_reader)
 
     for values_data in values_file_reader:
-        line_no +=1
+        line_no += 1
         if len(values_data) != key_count_in_values_file:
-            raise SystemExit("\nOops...Number of values is not equal to number of keys in file: %s at line No:%s\n"\
-            % (str(input_values_file), str(line_no)))
+            raise SystemExit("\nOops...Number of values is not equal to number of keys in file: %s at line No:%s\n"
+                             % (str(input_values_file), str(line_no)))
 
 
 def verify_keys_exist(values_file_keys, input_config_file):
@@ -62,18 +66,16 @@ def verify_keys_exist(values_file_keys, input_config_file):
             else:
                 keys_missing.append([config_data[0], line_no])
 
-
     if keys_missing:
         for key, line_no in keys_missing:
-            print("Key:`", str(key), "` at line no:", str(line_no),\
-            " in config file is not found in values file.")
+            print("Key:`", str(key), "` at line no:", str(line_no),
+                  " in config file is not found in values file.")
         config_file.close()
         raise SystemExit(1)
 
     config_file.close()
 
 
-
 def verify_datatype_encoding(input_config_file):
     """ Verify datatype and encodings from config file is valid
     """
@@ -84,15 +86,14 @@ def verify_datatype_encoding(input_config_file):
     config_file = open(input_config_file,'r')
     config_file_reader = csv.reader(config_file, delimiter=',')
     for config_data in config_file_reader:
-        line_no+=1
+        line_no += 1
         if config_data[1] not in valid_datatypes:
-            raise SystemExit("Oops...config file: %s has invalid datatype at line no:%s\n`" \
-            % (str(input_config_file), str(line_no)))
+            raise SystemExit("Oops...config file: %s has invalid datatype at line no:%s\n`"
+                             % (str(input_config_file), str(line_no)))
         if 'namespace' not in config_data:
             if config_data[2] not in valid_encodings:
-                raise SystemExit("Oops...config file: %s has invalid encoding at line no:%s\n`" \
-                % (str(input_config_file), str(line_no)))
-
+                raise SystemExit("Oops...config file: %s has invalid encoding at line no:%s\n`"
+                                 % (str(input_config_file), str(line_no)))
 
 
 def verify_file_data_count(input_config_file, keys_repeat):
@@ -105,8 +106,8 @@ def verify_file_data_count(input_config_file, keys_repeat):
     for line in config_file_reader:
         line_no += 1
         if len(line) != 3 and line[0] not in keys_repeat:
-            raise SystemExit("Oops...data missing in config file at line no:%s <format needed:key,type,encoding>\n" \
-            % str(line_no) )
+            raise SystemExit("Oops...data missing in config file at line no:%s <format needed:key,type,encoding>\n"
+                             % str(line_no))
     config_file.close()
 
 
@@ -131,9 +132,8 @@ def verify_data_in_file(input_config_file, input_values_file, config_file_keys,
 
         verify_values_exist(input_values_file, keys_in_values_file)
 
-    except StandardError as std_err:
-        print(std_err)
-    except:
+    except Exception as err:
+        print(err)
         raise
 
 
@@ -180,8 +180,8 @@ def add_config_data_per_namespace(input_config_file):
     return config_data_to_write
 
 
-def get_fileid_val(file_identifier, keys_in_config_file, keys_in_values_file,\
-values_data_line, key_value_data, fileid_value):
+def get_fileid_val(file_identifier, keys_in_config_file, keys_in_values_file,
+                   values_data_line, key_value_data, fileid_value):
     """ Get file identifier value
     """
     file_id_found = False
@@ -224,7 +224,6 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
                     del key_value_pair[0]
                     output_file_writer.writerow(data_to_write)
 
-
     # Set index to start of file
     target_csv_file.seek(0)
 
@@ -241,11 +240,9 @@ def create_dir(filetype, output_dir_path):
     return output_target_dir
 
 
-def set_repeat_value(total_keys_repeat, keys, csv_file):
+def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename):
     key_val_pair = []
     key_repeated = []
-    filename, file_ext = os.path.splitext(csv_file)
-    target_filename = filename + "_created" + file_ext
     with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
         csv_file_reader = csv.reader(read_from, delimiter=',')
         headers = next(csv_file_reader)
@@ -259,7 +256,6 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
         for row in csv_file_reader:
             index = -1
             key_val_new = list(zip_longest(keys, row))
-
             key_val_pair = total_keys_values[:]
             key_repeated = total_keys_repeat[:]
             while key_val_new and key_repeated:
@@ -273,37 +269,31 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
                 del key_val_new[0]
                 del key_val_pair[0]
 
-
     return target_filename
 
 
-def main(input_config_file=None,input_values_file=None,target_file_name_prefix=None,\
-file_identifier=None,output_dir_path=None):
+def main(input_config_file=None,input_values_file=None,target_file_name_prefix=None,
+         file_identifier=None,output_dir_path=None,part_size=None,input_version=None,
+         input_is_keygen=None,input_is_encrypt=None,input_is_keyfile=None):
     try:
-        if all(arg is None for arg in [input_config_file,input_values_file,target_file_name_prefix,\
-            file_identifier,output_dir_path]):
+        if all(arg is None for arg in [input_config_file,input_values_file,target_file_name_prefix,
+                                       file_identifier,output_dir_path]):
             parser = argparse.ArgumentParser(prog='./mfg_gen.py',
                                              description="Create binary files from input config and values file",
                                              formatter_class=argparse.RawDescriptionHelpFormatter)
 
-            parser.add_argument("--size",
-                                dest='part_size',
-                                required=True,
-                                help='Size of NVS Partition in bytes (must be multiple of 4096)')
-
             parser.add_argument('--conf',
                                 dest='config_file',
-                                required=True,
-                                help='the input configuration csv file')
+                                help='the input configuration csv file',
+                                default=None)
 
             parser.add_argument('--values',
                                 dest='values_file',
-                                required=True,
-                                help='the input values csv file')
+                                help='the input values csv file',
+                                default=None)
 
             parser.add_argument('--prefix',
                                 dest='prefix',
-                                required=True,
                                 help='the unique name as each filename prefix')
 
             parser.add_argument('--fileid',
@@ -313,30 +303,80 @@ file_identifier=None,output_dir_path=None):
 
             parser.add_argument('--outdir',
                                 dest='outdir',
-                                default='./',
+                                default=os.getcwd(),
                                 help='the output directory to store the files created\
                                 (Default: current directory)')
 
-            args = parser.parse_args()
+            parser.add_argument("--size",
+                                dest='part_size',
+                                help='Size of NVS Partition in bytes (must be multiple of 4096)')
+
+            parser.add_argument("--version",
+                                dest="version",
+                                help='Set version. Default: v2',
+                                choices=['v1','v2'],
+                                default='v2',
+                                type=str.lower)
+
+            parser.add_argument("--keygen",
+                                dest="keygen",
+                                help='Generate keys for encryption. Default: false',
+                                choices=['true','false'],
+                                default='false',
+                                type=str.lower)
+
+            parser.add_argument("--encrypt",
+                                dest="encrypt",
+                                help='Set encryption mode. Default: false',
+                                choices=['true','false'],
+                                default='false',
+                                type=str.lower)
+
+            parser.add_argument("--keyfile",
+                                dest="keyfile",
+                                help='File having key for encryption (Applicable only if encryption mode is true)',
+                                default=None)
 
-            # Verify if output_dir_path argument is given then output directory exists
-            if not os.path.isdir(args.outdir):
-                parser.error('--outdir ' + args.outdir + ' does not exist...')
+            args = parser.parse_args()
 
-            # Add '/' to outdir if it is not present
-            if not args.outdir.endswith('/'):
-                args.outdir = args.outdir + '/'
+            args.outdir = os.path.join(args.outdir, '')
 
-            input_part_size = args.part_size
             input_config_file = args.config_file
             input_values_file = args.values_file
             target_file_name_prefix = args.prefix
             output_dir_path = args.outdir
+            part_size = args.part_size
+            input_version = args.version
+            input_is_keygen = args.keygen
+            input_is_encrypt = args.encrypt
+            input_is_keyfile = args.keyfile
             file_identifier = ''
+            print_arg_str = "Invalid.\nTo generate binary --conf, --values, --prefix and --size arguments are mandatory.\
+            \nTo generate encryption keys --keygen argument is mandatory."
+            print_encrypt_arg_str = "Missing parameter. Enter --keygen or --keyfile."
 
             if args.fileid:
                 file_identifier = args.fileid
 
+            if input_config_file and input_is_encrypt.lower() == 'true' and input_is_keygen.lower() == 'true' and input_is_keyfile:
+                sys.exit('Invalid. Cannot provide both --keygen and --keyfile argument together.')
+
+            nvs_partition_gen.check_input_args(input_config_file, input_values_file, part_size, input_is_keygen,
+                                               input_is_encrypt, input_is_keyfile, input_version, print_arg_str,
+                                               print_encrypt_arg_str, output_dir_path)
+
+            if not input_config_file and input_is_keygen:
+                if input_is_encrypt == 'true':
+                    sys.exit("Invalid.\nOnly --keyfile and --outdir arguments allowed.\n")
+                # Generate Key Only
+                nvs_partition_gen.nvs_part_gen(input_filename=input_config_file, output_filename=input_values_file,
+                                               input_part_size=part_size, is_key_gen=input_is_keygen,
+                                               encrypt_mode=input_is_encrypt, key_file=input_is_keyfile,
+                                               version_no=input_version, output_dir=output_dir_path)
+                exit(0)
+
+            if not (input_config_file and input_values_file and target_file_name_prefix and part_size):
+                sys.exit(print_arg_str)
 
         keys_in_values_file = []
         keys_in_config_file = []
@@ -344,12 +384,12 @@ file_identifier=None,output_dir_path=None):
         key_value_data = []
         csv_file_list = []
         keys_repeat = []
-        is_keys_missing = True
-        file_id_found = False
         is_empty_line = False
         files_created = False
         file_identifier_value = '0'
         output_target_dir = ''
+        target_values_file = None
+        output_file_prefix = None
 
         # Verify config file is not empty
         if os.stat(input_config_file).st_size == 0:
@@ -357,7 +397,7 @@ file_identifier=None,output_dir_path=None):
 
         # Verify values file is not empty
         if os.stat(input_values_file).st_size == 0:
-            raise SystemExit("Oops...values file: %s is empty." % input_values_file )
+            raise SystemExit("Oops...values file: %s is empty." % input_values_file)
 
         # Verify config file does not have empty lines
         csv_config_file = open(input_config_file,'r')
@@ -372,17 +412,15 @@ file_identifier=None,output_dir_path=None):
                         is_empty_line = False
                         break
                 if is_empty_line:
-                    raise SystemExit("Oops...config file: %s cannot have empty lines. " % input_config_file )
+                    raise SystemExit("Oops...config file: %s cannot have empty lines. " % input_config_file)
                 if not config_data:
-                    raise SystemExit("Oops...config file: %s cannot have empty lines." % input_config_file )
+                    raise SystemExit("Oops...config file: %s cannot have empty lines." % input_config_file)
 
             csv_config_file.seek(0)
 
             # Extract keys from config file
             for config_data in config_file_reader:
-                if 'namespace' in config_data:
-                    namespace = config_data[0]
-                else:
+                if 'namespace' not in config_data:
                     keys_in_config_file.append(config_data[0])
                 if 'REPEAT' in config_data:
                     keys_repeat.append(config_data[0])
@@ -394,8 +432,6 @@ file_identifier=None,output_dir_path=None):
             csv_config_file.close()
 
         is_empty_line = False
-
-
         # Verify values file does not have empty lines
         csv_values_file = open(input_values_file, 'r')
         try:
@@ -409,9 +445,9 @@ file_identifier=None,output_dir_path=None):
                         is_empty_line = False
                         break
                 if is_empty_line:
-                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
+                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file)
                 if not values_data:
-                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
+                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file)
 
             csv_values_file.seek(0)
 
@@ -428,12 +464,11 @@ file_identifier=None,output_dir_path=None):
         # Verify file identifier exists in values file
         if file_identifier:
             if file_identifier not in keys_in_values_file:
-                raise SystemExit('Oops...target_file_identifier: %s does not exist in values file.\n' % file_identifier )
-
+                raise SystemExit('Oops...target_file_identifier: %s does not exist in values file.\n' % file_identifier)
 
         # Verify data in the input_config_file and input_values_file
-        verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,\
-        keys_in_values_file, keys_repeat)
+        verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,
+                            keys_in_values_file, keys_repeat)
 
         # Add config data per namespace to `config_data_to_write` list
         config_data_to_write = add_config_data_per_namespace(input_config_file)
@@ -443,20 +478,23 @@ file_identifier=None,output_dir_path=None):
                 values_file_reader = csv.reader(csv_values_file, delimiter=',')
                 keys = next(values_file_reader)
 
-            target_values_file = set_repeat_value(keys_repeat, keys, input_values_file)
+            filename, file_ext = os.path.splitext(input_values_file)
+            target_filename = filename + "_created" + file_ext
+            if keys_repeat:
+                target_values_file = set_repeat_value(keys_repeat, keys, input_values_file, target_filename)
+            else:
+                target_values_file = input_values_file
 
             csv_values_file = open(target_values_file, 'r')
 
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
             next(values_file_reader)
-
-
             for values_data_line in values_file_reader:
                 key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
 
                 # Get file identifier value from values file
-                file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \
-                keys_in_values_file, values_data_line, key_value_data, file_identifier_value)
+                file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file,
+                                                       keys_in_values_file, values_data_line, key_value_data, file_identifier_value)
 
                 key_value_pair = key_value_data[:]
 
@@ -468,7 +506,7 @@ file_identifier=None,output_dir_path=None):
                 csv_file_list.append(csv_filename)
                 output_csv_file = output_target_dir + csv_filename
                 if os.path.isfile(output_csv_file):
-                    raise SystemExit("Target csv file: %s already exists.`" % output_csv_file )
+                    raise SystemExit("Target csv file: %s already exists.`" % output_csv_file)
 
                 # Add values corresponding to each key to csv target file
                 add_data_to_file(config_data_to_write, key_value_pair, output_csv_file)
@@ -477,16 +515,20 @@ file_identifier=None,output_dir_path=None):
                 output_target_dir = create_dir("bin/", output_dir_path)
 
                 # Verify if output bin file does not exist
-                output_bin_file = output_target_dir + target_file_name_prefix + "-" +\
-                file_identifier_value + ".bin"
+                # todo for keys
+                output_file_prefix = target_file_name_prefix + "-" + file_identifier_value
+                output_bin_file = output_target_dir + output_file_prefix + ".bin"
                 if os.path.isfile(output_bin_file):
-                    raise SystemExit("Target csv file: %s already exists.`" % output_bin_file )
+                    raise SystemExit("Target csv file: %s already exists.`" % output_bin_file)
 
                 # Create output csv and bin file
+                if input_is_keygen.lower() == 'true' and input_is_keyfile:
+                    input_is_keyfile = os.path.basename(input_is_keyfile)
+                nvs_partition_gen.nvs_part_gen(input_filename=output_csv_file, output_filename=output_bin_file,
+                                               input_part_size=part_size, is_key_gen=input_is_keygen,
+                                               encrypt_mode=input_is_encrypt, key_file=input_is_keyfile,
+                                               version_no=input_version, encr_key_prefix=output_file_prefix, output_dir=output_dir_path)
                 print("CSV Generated: ", str(output_csv_file))
-                nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file,\
-                                               input_size=input_part_size)
-                print("NVS Flash Binary Generated: ", str(output_bin_file))
 
                 files_created = True
 
@@ -496,14 +538,13 @@ file_identifier=None,output_dir_path=None):
             exit(1)
         finally:
             csv_values_file.close()
-
-
-        return csv_file_list, files_created
+        return csv_file_list, files_created, target_values_file
 
     except ValueError as err:
         print(err)
-    except:
+    except Exception:
         raise
 
+
 if __name__ == "__main__":
     main()

tools/mass_mfg/samples/sample_config.csv

@@ -1,4 +1,15 @@
-app,namespace,  
-firmware_key,data,hex2bin
-serial_no,data,i32,REPEAT
-device_no,data,i32
+dummyNamespace,namespace,
+dummyU8Key,data,u8
+dummyI8Key,data,i8
+dummyU16Key,data,u16
+dummyU32Key,data,u32
+dummyI32Key,data,i32,REPEAT
+dummyStringKey,data,string
+dummyHex2BinKey,data,hex2bin
+dummyBase64Key,data,base64
+hexFileKey,file,hex2bin
+base64FileKey,file,base64
+stringFileKey,file,string
+blobFileAKey,file,binary
+blobFileBKey,file,binary
+binFileKey,file,binary

tools/mass_mfg/samples/sample_values.csv

@@ -1,6 +0,0 @@
-id,firmware_key,serial_no,device_no
-1,1a2b3c4d5e6faabb,111,101
-2,1a2b3c4d5e6fccdd,,102
-3,1a2b3c4d5e6feeff,,103
-4,1a2b3c4d5e6faabb,,104
-5,1a2b3c4d5e6feedd,,105

tools/mass_mfg/samples/sample_values_multipage_blob.csv

@@ -0,0 +1,4 @@
+id,dummyU8Key,dummyI8Key,dummyU16Key,dummyU32Key,dummyI32Key,dummyStringKey,dummyHex2BinKey,dummyBase64Key,hexFileKey,base64FileKey,stringFileKey,blobFileAKey,blobFileBKey,binFileKey
+1,127,-128,32768,4294967295,-2147483648,0A:0B:0C:0D:0E:0F,010203abcdef,MTIzYWJj,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_multipage_blob.bin
+2,126,-127,32767,4294967294,,A0:B0:C0:D0:E0:F0,102030abcdef,MTIzYWFh,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_multipage_blob.bin
+3,125,-126,32766,4294967293,,00:B3:C4:BD:E2:0F,010203efcdab,MTIzYmJi,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_multipage_blob.bin

tools/mass_mfg/samples/sample_values_singlepage_blob.csv

@@ -0,0 +1,4 @@
+id,dummyU8Key,dummyI8Key,dummyU16Key,dummyU32Key,dummyI32Key,dummyStringKey,dummyHex2BinKey,dummyBase64Key,hexFileKey,base64FileKey,stringFileKey,blobFileAKey,blobFileBKey,binFileKey
+1,127,-128,32768,4294967295,-2147483648,0A:0B:0C:0D:0E:0F,010203abcdef,MTIzYWJj,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
+2,126,-127,32767,4294967294,,A0:B0:C0:D0:E0:F0,102030abcdef,MTIzYWFh,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
+3,125,-126,32766,4294967293,,00:B3:C4:BD:E2:0F,010203efcdab,MTIzYmJi,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin

tools/mass_mfg/testdata/sample.base64

@@ -0,0 +1 @@
+AQIDBAUGBwgJq83v

tools/mass_mfg/testdata/sample.hex

@@ -0,0 +1 @@
+0123456789abcdef

tools/mass_mfg/testdata/sample.txt

@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

tools/mass_mfg/testdata/sample_blob.bin

@@ -0,0 +1 @@
+start0000000000000000000000start0123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef0000000000000000end00000000000000000000000000end

tools/mass_mfg/testdata/sample_singlepage_blob.bin

@@ -0,0 +1 @@
+start0000000000000000000000start0123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef0000000000000000