mfg_util: Update file handling to maintain line endings across various OS

Fixes: https://github.com/espressif/esp-idf/issues/4888

tools/mass_mfg/mfg_gen.py

@@ -16,11 +16,10 @@
 #
 
 from __future__ import print_function
-from builtins import range
 from future.moves.itertools import zip_longest
+from io import open
 import sys
 import os
-import csv
 import argparse
 import distutils.dir_util
 
@@ -32,83 +31,74 @@ except Exception as e:
     sys.exit("Please check IDF_PATH")
 
 
-def verify_values_exist(input_values_file, keys_in_values_file):
+def verify_values_exist(input_values_file, values_file_data, key_count_in_values_file, line_no=1):
     """ Verify all keys have corresponding values in values file
     """
-    line_no = 1
-    key_count_in_values_file = len(keys_in_values_file)
+    if len(values_file_data) != key_count_in_values_file:
+        raise SystemExit("\nError: Number of values is not equal to number of keys in file: %s at line No:%s\n"
+                         % (str(input_values_file), str(line_no)))
 
-    values_file = open(input_values_file, 'r')
-    values_file_reader = csv.reader(values_file, delimiter=',')
-    next(values_file_reader)
 
-    for values_data in values_file_reader:
-        line_no += 1
-        if len(values_data) != key_count_in_values_file:
-            raise SystemExit("\nError: Number of values is not equal to number of keys in file: %s at line No:%s\n"
-                             % (str(input_values_file), str(line_no)))
-
-
-def verify_keys_exist(values_file_keys, input_config_file):
+def verify_keys_exist(values_file_keys, config_file_data):
     """ Verify all keys from config file are present in values file
     """
     keys_missing = []
 
-    config_file = open(input_config_file,'r')
-    config_file_reader = csv.reader(config_file, delimiter=',')
-    for line_no, config_data in enumerate(config_file_reader,1):
-        if 'namespace' not in config_data:
+    for line_no, config_data in enumerate(config_file_data,1):
+        if not isinstance(config_data, str):
+            config_data = config_data.encode('utf-8')
+        config_data_line = config_data.strip().split(',')
+        if 'namespace' not in config_data_line:
             if values_file_keys:
-                if config_data[0] == values_file_keys[0]:
+                if config_data_line[0] == values_file_keys[0]:
                     del values_file_keys[0]
                 else:
-                    keys_missing.append([config_data[0], line_no])
+                    keys_missing.append([config_data_line[0], line_no])
             else:
-                keys_missing.append([config_data[0], line_no])
+                keys_missing.append([config_data_line[0], line_no])
 
     if keys_missing:
         for key, line_no in keys_missing:
             print("Key:`", str(key), "` at line no:", str(line_no),
                   " in config file is not found in values file.")
-        config_file.close()
         raise SystemExit(1)
 
-    config_file.close()
 
-
-def verify_datatype_encoding(input_config_file):
+def verify_datatype_encoding(input_config_file, config_file_data):
     """ Verify datatype and encodings from config file is valid
     """
     valid_encodings = ["string", "binary", "hex2bin","u8", "i8", "u16", "u32", "i32","base64"]
     valid_datatypes = ["file","data","namespace"]
     line_no = 0
 
-    config_file = open(input_config_file,'r')
-    config_file_reader = csv.reader(config_file, delimiter=',')
-    for config_data in config_file_reader:
+    for data in config_file_data:
         line_no += 1
-        if config_data[1] not in valid_datatypes:
-            raise SystemExit("Error: config file: %s has invalid datatype at line no:%s\n`"
+        if not isinstance(data, str):
+            data = data.encode('utf-8')
+        line = data.strip().split(',')
+        if line[1] not in valid_datatypes:
+            raise SystemExit("Error: config file: %s has invalid datatype at line no:%s\n"
                              % (str(input_config_file), str(line_no)))
-        if 'namespace' not in config_data:
-            if config_data[2] not in valid_encodings:
-                raise SystemExit("Error: config file: %s has invalid encoding at line no:%s\n`"
+        if 'namespace' not in line:
+            if line[2] not in valid_encodings:
+                raise SystemExit("Error: config file: %s has invalid encoding at line no:%s\n"
                                  % (str(input_config_file), str(line_no)))
 
 
-def verify_file_data_count(input_config_file, keys_repeat):
+def verify_file_data_count(cfg_file_data, keys_repeat):
     """ Verify count of data on each line in config file is equal to 3
     (as format must be: <key,type and encoding>)
     """
     line_no = 0
-    config_file = open(input_config_file, 'r')
-    config_file_reader = csv.reader(config_file, delimiter=',')
-    for line in config_file_reader:
+
+    for data in cfg_file_data:
         line_no += 1
+        if not isinstance(data, str):
+            data = data.encode('utf-8')
+        line = data.strip().split(',')
         if len(line) != 3 and line[0] not in keys_repeat:
             raise SystemExit("Error: data missing in config file at line no:%s <format needed:key,type,encoding>\n"
                              % str(line_no))
-    config_file.close()
 
 
 def verify_data_in_file(input_config_file, input_values_file, config_file_keys, keys_in_values_file, keys_repeat):
@@ -120,17 +110,35 @@ def verify_data_in_file(input_config_file, input_values_file, config_file_keys,
     """
     try:
         values_file_keys = []
-
-        verify_file_data_count(input_config_file, keys_repeat)
-
-        verify_datatype_encoding(input_config_file)
+        values_file_line = None
 
         # Get keys from values file present in config files
         values_file_keys = get_keys(keys_in_values_file, config_file_keys)
 
-        verify_keys_exist(values_file_keys, input_config_file)
-
-        verify_values_exist(input_values_file, keys_in_values_file)
+        with open(input_config_file, 'r', newline='\n') as cfg_file:
+            cfg_file_data = cfg_file.readlines()
+            verify_file_data_count(cfg_file_data, keys_repeat)
+            verify_datatype_encoding(input_config_file, cfg_file_data)
+            verify_keys_exist(values_file_keys, cfg_file_data)
+
+        with open(input_values_file, 'r', newline='\n') as values_file:
+            key_count_in_values_file = len(keys_in_values_file)
+            lineno = 0
+            # Read first keys(header) line
+            values_file_data = values_file.readline()
+            lineno += 1
+            while values_file_data:
+                # Read values line
+                values_file_line = values_file.readline()
+                if not isinstance(values_file_line, str):
+                    values_file_line = values_file_line.encode('utf-8')
+
+                values_file_data = values_file_line.strip().split(',')
+
+                lineno += 1
+                if len(values_file_data) == 1 and '' in values_file_data:
+                    break
+                verify_values_exist(input_values_file, values_file_data, key_count_in_values_file, line_no=lineno)
 
     except Exception as err:
         print(err)
@@ -141,9 +149,9 @@ def get_keys(keys_in_values_file, config_file_keys):
     """ Get keys from values file present in config file
     """
     values_file_keys = []
-    for key in range(len(keys_in_values_file)):
-        if keys_in_values_file[key] in config_file_keys:
-            values_file_keys.append(keys_in_values_file[key])
+    for key in keys_in_values_file:
+        if key in config_file_keys:
+            values_file_keys.append(key)
 
     return values_file_keys
 
@@ -154,29 +162,30 @@ def add_config_data_per_namespace(input_config_file):
     config_data_to_write = []
     config_data_per_namespace = []
 
-    csv_config_file = open(input_config_file,'r')
-    config_file_reader = csv.reader(csv_config_file, delimiter=',')
+    with open(input_config_file, 'r', newline='\n') as cfg_file:
+        config_data = cfg_file.readlines()
 
     # `config_data_per_namespace` is added to `config_data_to_write` list after reading next namespace
-    for config_data in config_file_reader:
-        if 'REPEAT' in config_data:
-            config_data.remove('REPEAT')
-        if 'namespace' in config_data:
+    for data in config_data:
+        if not isinstance(data, str):
+            data = data.encode('utf-8')
+        cfg_data = data.strip().split(',')
+        if 'REPEAT' in cfg_data:
+            cfg_data.remove('REPEAT')
+        if 'namespace' in cfg_data:
             if config_data_per_namespace:
                 config_data_to_write.append(config_data_per_namespace)
                 config_data_per_namespace = []
-                config_data_per_namespace.append(config_data)
+                config_data_per_namespace.append(cfg_data)
             else:
-                config_data_per_namespace.append(config_data)
+                config_data_per_namespace.append(cfg_data)
         else:
-            config_data_per_namespace.append(config_data)
+            config_data_per_namespace.append(cfg_data)
 
     # `config_data_per_namespace` is added to `config_data_to_write` list as EOF is reached
     if (not config_data_to_write) or (config_data_to_write and config_data_per_namespace):
         config_data_to_write.append(config_data_per_namespace)
 
-    csv_config_file.close()
-
     return config_data_to_write
 
 
@@ -202,18 +211,21 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
     """
     header = ['key', 'type', 'encoding', 'value']
     data_to_write = []
+    newline = u"\n"
 
-    target_csv_file = open(output_csv_file, 'w')
-
-    output_file_writer = csv.writer(target_csv_file, delimiter=',')
-    output_file_writer.writerow(header)
+    target_csv_file = open(output_csv_file, 'w', newline=None)
 
+    line_to_write = u",".join(header)
+    target_csv_file.write(line_to_write)
+    target_csv_file.write(newline)
     for namespace_config_data in config_data_to_write:
         for data in namespace_config_data:
             data_to_write = data[:]
             if 'namespace' in data:
                 data_to_write.append('')
-                output_file_writer.writerow(data_to_write)
+                line_to_write = u",".join(data_to_write)
+                target_csv_file.write(line_to_write)
+                target_csv_file.write(newline)
             else:
                 key = data[0]
                 while key not in key_value_pair[0]:
@@ -222,7 +234,9 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
                     value = key_value_pair[0][1]
                     data_to_write.append(value)
                     del key_value_pair[0]
-                    output_file_writer.writerow(data_to_write)
+                    line_to_write = u",".join(data_to_write)
+                    target_csv_file.write(line_to_write)
+                    target_csv_file.write(newline)
 
     # Set index to start of file
     target_csv_file.seek(0)
@@ -242,17 +256,24 @@ def create_dir(filetype, output_dir_path):
 def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename):
     key_val_pair = []
     key_repeated = []
-    with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
-        csv_file_reader = csv.reader(read_from, delimiter=',')
-        headers = next(csv_file_reader)
-        values = next(csv_file_reader)
-        csv_file_writer = csv.writer(write_to, delimiter=',')
-        csv_file_writer.writerow(headers)
-        csv_file_writer.writerow(values)
+    line = None
+    newline = u"\n"
+    with open(csv_file, 'r', newline=None) as read_from, open(target_filename,'w', newline=None) as write_to:
+        headers = read_from.readline()
+        values = read_from.readline()
+        write_to.write(headers)
+        write_to.write(values)
+        if not isinstance(values, str):
+            values = values.encode('utf-8')
+        values = values.strip().split(',')
         total_keys_values = list(zip_longest(keys, values))
 
         # read new data, add value if key has repeat tag, write to new file
-        for row in csv_file_reader:
+        line = read_from.readline()
+        if not isinstance(line, str):
+            line = line.encode('utf-8')
+        row = line.strip().split(',')
+        while row:
             index = -1
             key_val_new = list(zip_longest(keys, row))
             key_val_pair = total_keys_values[:]
@@ -266,7 +287,18 @@ def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename):
                     del key_repeated[0]
                 del key_val_new[0]
                 del key_val_pair[0]
-            csv_file_writer.writerow(row)
+
+            line_to_write = u",".join(row)
+            write_to.write(line_to_write)
+            write_to.write(newline)
+
+            # Read next line
+            line = read_from.readline()
+            if not isinstance(line, str):
+                line = line.encode('utf-8')
+            row = line.strip().split(',')
+            if len(row) == 1 and '' in row:
+                break
 
     return target_filename
 
@@ -275,15 +307,19 @@ def create_intermediate_csv(args, keys_in_config_file, keys_in_values_file, keys
     file_identifier_value = '0'
     csv_str = 'csv'
     bin_str = 'bin'
+    line = None
     set_output_keyfile = False
 
     # Add config data per namespace to `config_data_to_write` list
     config_data_to_write = add_config_data_per_namespace(args.conf)
 
     try:
-        with open(args.values, 'r') as csv_values_file:
-            values_file_reader = csv.reader(csv_values_file, delimiter=',')
-            keys = next(values_file_reader)
+        with open(args.values, 'r', newline=None) as csv_values_file:
+            # first line must be keys in file
+            line = csv_values_file.readline()
+            if not isinstance(line, str):
+                line = line.encode('utf-8')
+            keys = line.strip().split(',')
 
         filename, file_ext = os.path.splitext(args.values)
         target_filename = filename + "_created" + file_ext
@@ -292,10 +328,10 @@ def create_intermediate_csv(args, keys_in_config_file, keys_in_values_file, keys
         else:
             target_values_file = args.values
 
-        csv_values_file = open(target_values_file, 'r')
+        csv_values_file = open(target_values_file, 'r', newline=None)
 
-        values_file_reader = csv.reader(csv_values_file, delimiter=',')
-        next(values_file_reader)
+        # Read header line
+        csv_values_file.readline()
 
         # Create new directory(if doesn't exist) to store csv file generated
         output_csv_target_dir = create_dir(csv_str, args.outdir)
@@ -304,7 +340,12 @@ def create_intermediate_csv(args, keys_in_config_file, keys_in_values_file, keys
         if args.keygen:
             set_output_keyfile = True
 
-        for values_data_line in values_file_reader:
+        line = csv_values_file.readline()
+        if not isinstance(line, str):
+            line = line.encode('utf-8')
+        values_data_line = line.strip().split(',')
+
+        while values_data_line:
             key_value_data = list(zip_longest(keys_in_values_file, values_data_line))
 
             # Get file identifier value from values file
@@ -340,6 +381,14 @@ def create_intermediate_csv(args, keys_in_config_file, keys_in_values_file, keys
             else:
                 nvs_partition_gen.generate(args)
 
+            # Read next line
+            line = csv_values_file.readline()
+            if not isinstance(line, str):
+                line = line.encode('utf-8')
+            values_data_line = line.strip().split(',')
+            if len(values_data_line) == 1 and '' in values_data_line:
+                break
+
         print("\nFiles generated in %s ..." % args.outdir)
 
     except Exception as e:
@@ -349,25 +398,21 @@ def create_intermediate_csv(args, keys_in_config_file, keys_in_values_file, keys
         csv_values_file.close()
 
 
-def verify_empty_lines_exist(args, input_file):
-    input_file_reader = csv.reader(input_file, delimiter=',')
-    for file_data in input_file_reader:
-        for data in file_data:
-            if len(data.strip()) == 0:
-                raise SystemExit("Error: config file: %s cannot have empty lines. " % args.conf)
-            else:
-                break
-        if not file_data:
-            raise SystemExit("Error: config file: %s cannot have empty lines." % args.conf)
+def verify_empty_lines_exist(file_name, input_file_data):
+    for data in input_file_data:
+        if not isinstance(data, str):
+            data = data.encode('utf-8')
+        cfg_data = data.strip().split(',')
 
-    input_file.seek(0)
-    return input_file_reader
+        if len(cfg_data) == 1 and '' in cfg_data:
+            raise SystemExit("Error: file: %s cannot have empty lines. " % file_name)
 
 
 def verify_file_format(args):
     keys_in_config_file = []
     keys_in_values_file = []
     keys_repeat = []
+    file_data_keys = None
 
     # Verify config file is not empty
     if os.stat(args.conf).st_size == 0:
@@ -378,25 +423,41 @@ def verify_file_format(args):
         raise SystemExit("Error: values file: %s is empty." % args.values)
 
     # Verify config file does not have empty lines
-    with open(args.conf, 'r') as csv_config_file:
+    with open(args.conf, 'r', newline='\n') as csv_config_file:
         try:
-            config_file_reader = verify_empty_lines_exist(args, csv_config_file)
-            # Extract keys from config file
-            for config_data in config_file_reader:
-                if 'namespace' not in config_data:
-                    keys_in_config_file.append(config_data[0])
-                if 'REPEAT' in config_data:
-                    keys_repeat.append(config_data[0])
+            file_data = csv_config_file.readlines()
+            verify_empty_lines_exist(args.conf, file_data)
 
+            csv_config_file.seek(0)
+            # Extract keys from config file
+            for data in file_data:
+                if not isinstance(data, str):
+                    data = data.encode('utf-8')
+                line_data = data.strip().split(',')
+                if 'namespace' not in line_data:
+                    keys_in_config_file.append(line_data[0])
+                if 'REPEAT' in line_data:
+                    keys_repeat.append(line_data[0])
         except Exception as e:
             print(e)
 
     # Verify values file does not have empty lines
-    with open(args.values, 'r') as csv_values_file:
+    with open(args.values, 'r', newline='\n') as csv_values_file:
         try:
-            values_file_reader = verify_empty_lines_exist(args, csv_values_file)
-            # Extract keys from values file
-            keys_in_values_file = next(values_file_reader)
+            # Extract keys from values file (first line of file)
+            file_data = [csv_values_file.readline()]
+
+            file_data_keys = file_data[0]
+            if not isinstance(file_data_keys, str):
+                file_data_keys = file_data_keys.encode('utf-8')
+
+            keys_in_values_file = file_data_keys.strip().split(',')
+
+            while file_data:
+                verify_empty_lines_exist(args.values, file_data)
+                file_data = [csv_values_file.readline()]
+                if '' in file_data:
+                    break
 
         except Exception as e:
             print(e)
@@ -428,6 +489,7 @@ def generate(args):
     if (args.keygen or args.inputkey):
         encryption_enabled = True
         print("\nGenerating encrypted NVS binary images...")
+
     # Create intermediate csv file
     create_intermediate_csv(args, keys_in_config_file, keys_in_values_file,
                             keys_repeat, is_encr=encryption_enabled)