tools: Introduce support for blank lines in config and value files for mfg utility

Some users have requested this feature.

In order to avoid complete refactoring or introducing more code complexity, a design choice to create temporary files without blank lines is made.

Additionally, an extension check is added and there are multiple smaller code style and structure improvements.

Closes https://github.com/espressif/esp-idf/issues/8421

components/nvs_flash/test_nvs_host/test_nvs.cpp

@@ -2997,6 +2997,87 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
 
 }
 
+TEST_CASE("check and read data from partition generated via manufacturing utility with blank lines in csv files and multipage blob support disabled", "[mfg_gen]")
+{
+    int childpid = fork();
+    int status;
+
+    if (childpid == 0) {
+        exit(execlp("bash", "bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test && \
+                    cp -rf ../../../tools/mass_mfg/testdata mfg_testdata && \
+                    cp -rf ../nvs_partition_generator/testdata . && \
+                    mkdir -p ../../../tools/mass_mfg/host_test", NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) == 0);
+
+        childpid = fork();
+        if (childpid == 0) {
+            exit(execlp("python", "python",
+                        "../../../tools/mass_mfg/mfg_gen.py",
+                        "generate",
+                        "../../../tools/mass_mfg/samples/sample_config_blank_lines.csv",
+                        "../../../tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv",
+                        "Test",
+                        "0x3000",
+                        "--outdir",
+                        "../../../tools/mass_mfg/host_test",
+                        "--version",
+                        "1",NULL));
+
+        } else {
+            CHECK(childpid > 0);
+            waitpid(childpid, &status, 0);
+            CHECK(WEXITSTATUS(status) == 0);
+
+            childpid = fork();
+            if (childpid == 0) {
+                exit(execlp("python", "python",
+                            "../nvs_partition_generator/nvs_partition_gen.py",
+                            "generate",
+                            "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
+                            "../nvs_partition_generator/Test-1-partition.bin",
+                            "0x3000",
+                            "--version",
+                            "1",NULL));
+
+            } else {
+                CHECK(childpid > 0);
+                waitpid(childpid, &status, 0);
+                CHECK(WEXITSTATUS(status) == 0);
+
+            }
+
+        }
+
+    }
+
+    SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
+    check_nvs_part_gen_args_mfg(&emu1, "test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL);
+
+    SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin");
+    check_nvs_part_gen_args_mfg(&emu2, "test", 3, "testdata/sample_singlepage_blob.bin", false, NULL);
+
+
+    childpid = fork();
+    if (childpid == 0) {
+        exit(execlp("bash", " bash",
+                    "-c",
+                    "rm -rf ../../../tools/mass_mfg/host_test | \
+                    rm -rf mfg_testdata | \
+                    rm -rf testdata",NULL));
+    } else {
+        CHECK(childpid > 0);
+        waitpid(childpid, &status, 0);
+        CHECK(WEXITSTATUS(status) == 0);
+
+    }
+
+}
+
 TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support enabled", "[mfg_gen]")
 {
     int childpid = fork();

tools/mass_mfg/mfg_gen.py

@@ -23,6 +23,23 @@ except Exception as e:
     sys.exit('Please check IDF_PATH')
 
 
+def create_temp_files(args):
+    new_filenames = []
+    for filename in [args.conf, args.values]:
+        name, ext = os.path.splitext(filename)
+        new_filename = name + '_tmp' + ext
+        strip_blank_lines(filename, new_filename)
+        new_filenames.append(new_filename)
+    return new_filenames
+
+
+def strip_blank_lines(input_filename, output_filename):
+    with open(input_filename, 'r') as read_from, open(output_filename,'w', newline='') as write_to:
+        for line in read_from:
+            if not line.isspace():
+                write_to.write(line)
+
+
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
     """
@@ -237,10 +254,6 @@ def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename):
 
 def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=False):
     file_identifier_value = '0'
-    csv_str = 'csv'
-    bin_str = 'bin'
-    set_output_keyfile = False
-
     # Add config data per namespace to `config_data_to_write` list
     config_data_to_write = add_config_data_per_namespace(args.conf)
 
@@ -261,11 +274,9 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
             next(values_file_reader)
 
             # Create new directory(if doesn't exist) to store csv file generated
-            output_csv_target_dir = create_dir(csv_str, args.outdir)
+            output_csv_target_dir = create_dir('csv', args.outdir)
             # Create new directory(if doesn't exist) to store bin file generated
-            output_bin_target_dir = create_dir(bin_str, args.outdir)
-            if args.keygen:
-                set_output_keyfile = True
+            output_bin_target_dir = create_dir('bin', args.outdir)
 
             for values_data_line in values_file_reader:
                 key_value_data = list(zip_longest(keys_in_values_file, values_data_line))
@@ -276,7 +287,7 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
                 key_value_pair = key_value_data[:]
 
                 # Verify if output csv file does not exist
-                csv_filename = args.prefix + '-' + file_identifier_value + '.' + csv_str
+                csv_filename = args.prefix + '-' + file_identifier_value + '.' + 'csv'
                 output_csv_file = output_csv_target_dir + csv_filename
                 if os.path.isfile(output_csv_file):
                     raise SystemExit('Target csv file: %s already exists.`' % output_csv_file)
@@ -286,14 +297,14 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
                 print('\nCreated CSV file: ===>', output_csv_file)
 
                 # Verify if output bin file does not exist
-                bin_filename = args.prefix + '-' + file_identifier_value + '.' + bin_str
+                bin_filename = args.prefix + '-' + file_identifier_value + '.' + 'bin'
                 output_bin_file = output_bin_target_dir + bin_filename
                 if os.path.isfile(output_bin_file):
                     raise SystemExit('Target binary file: %s already exists.`' % output_bin_file)
 
                 args.input = output_csv_file
-                args.output = os.path.join(bin_str, bin_filename)
-                if set_output_keyfile:
+                args.output = os.path.join('bin', bin_filename)
+                if args.keygen:
                     args.keyfile = 'keys-' + args.prefix + '-' + file_identifier_value
 
                 if is_encr:
@@ -308,57 +319,38 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
         exit(1)
 
 
-def verify_empty_lines_exist(args, input_file):
-    input_file_reader = csv.reader(input_file, delimiter=',')
-    for file_data in input_file_reader:
-        for data in file_data:
-            if len(data.strip()) == 0:
-                raise SystemExit('Error: config file: %s cannot have empty lines. ' % args.conf)
-            else:
-                break
-        if not file_data:
-            raise SystemExit('Error: config file: %s cannot have empty lines.' % args.conf)
-
-    input_file.seek(0)
-    return input_file_reader
-
-
 def verify_file_format(args):
     keys_in_config_file = []
     keys_in_values_file = []
     keys_repeat = []
 
-    # Verify config file is not empty
+    # Verify files have csv extension
+    conf_name, conf_extension = os.path.splitext(args.conf)
+    if conf_extension != '.csv':
+        raise SystemExit('Error: config file: %s does not have the .csv extension.' % args.conf)
+    values_name, values_extension = os.path.splitext(args.values)
+    if values_extension != '.csv':
+        raise SystemExit('Error: values file: %s does not have the .csv extension.' % args.values)
+
+    # Verify files are not empty
     if os.stat(args.conf).st_size == 0:
         raise SystemExit('Error: config file: %s is empty.' % args.conf)
-
-    # Verify values file is not empty
     if os.stat(args.values).st_size == 0:
         raise SystemExit('Error: values file: %s is empty.' % args.values)
 
-    # Verify config file does not have empty lines
-    with open(args.conf, 'r') as csv_config_file:
-        try:
-            config_file_reader = verify_empty_lines_exist(args, csv_config_file)
-            # Extract keys from config file
-            for config_data in config_file_reader:
-                if 'namespace' not in config_data:
-                    keys_in_config_file.append(config_data[0])
-                if 'REPEAT' in config_data:
-                    keys_repeat.append(config_data[0])
-
-        except Exception as e:
-            print(e)
-
-    # Verify values file does not have empty lines
-    with open(args.values, 'r') as csv_values_file:
-        try:
-            values_file_reader = verify_empty_lines_exist(args, csv_values_file)
-            # Extract keys from values file
-            keys_in_values_file = next(values_file_reader)
-
-        except Exception as e:
-            print(e)
+    # Extract keys from config file
+    with open(args.conf, 'r') as config_file:
+        config_file_reader = csv.reader(config_file, delimiter=',')
+        for config_data in config_file_reader:
+            if 'namespace' not in config_data:
+                keys_in_config_file.append(config_data[0])
+            if 'REPEAT' in config_data:
+                keys_repeat.append(config_data[0])
+
+    # Extract keys from values file
+    with open(args.values, 'r') as values_file:
+        values_file_reader = csv.reader(values_file, delimiter=',')
+        keys_in_values_file = next(values_file_reader)
 
     # Verify file identifier exists in values file
     if args.fileid:
@@ -371,7 +363,9 @@ def verify_file_format(args):
 
 
 def generate(args):
-    args.outdir = os.path.join(args.outdir, '')
+    # Create work files with no blank lines
+    args.conf, args.values = create_temp_files(args)
+
     # Verify input config and values file format
     keys_in_config_file, keys_in_values_file, keys_repeat = verify_file_format(args)
 

tools/mass_mfg/samples/sample_config_blank_lines.csv

@@ -0,0 +1,29 @@
+dummyNamespace,namespace,
+
+
+dummyU8Key,data,u8
+
+dummyI8Key,data,i8
+
+dummyU16Key,data,u16
+
+
+dummyU32Key,data,u32
+
+dummyI32Key,data,i32,REPEAT
+
+dummyStringKey,data,string
+
+dummyHex2BinKey,data,hex2bin
+
+dummyBase64Key,data,base64
+hexFileKey,file,hex2bin
+base64FileKey,file,base64
+
+stringFileKey,file,string
+
+blobFileAKey,file,binary
+
+
+blobFileBKey,file,binary
+binFileKey,file,binary

tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv

@@ -0,0 +1,7 @@
+id,dummyU8Key,dummyI8Key,dummyU16Key,dummyU32Key,dummyI32Key,dummyStringKey,dummyHex2BinKey,dummyBase64Key,hexFileKey,base64FileKey,stringFileKey,blobFileAKey,blobFileBKey,binFileKey
+
+1,127,-128,32768,4294967295,-2147483648,0A:0B:0C:0D:0E:0F,010203abcdef,MTIzYWJj,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
+
+
+2,126,-127,32767,4294967294,,A0:B0:C0:D0:E0:F0,102030abcdef,MTIzYWFh,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
+3,125,-126,32766,4294967293,,00:B3:C4:BD:E2:0F,010203efcdab,MTIzYmJi,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin