Merge branch 'bugfix/app_partition_fallback' into 'master'

partition_table: Add parttool.py fallback option, some small cleanups

See merge request idf/esp-idf!2593

components/fatfs/test_fatfs_host/Makefile

@@ -72,7 +72,7 @@ $(COMPONENT_LIB): $(OBJ_FILES)
 lib: $(COMPONENT_LIB)
 
 partition_table.bin: partition_table.csv
-	python ../../partition_table/gen_esp32part.py --verify $< $@
+	python ../../partition_table/gen_esp32part.py $< $@
 
 $(TEST_PROGRAM): lib $(TEST_OBJ_FILES) $(SPI_FLASH_SIM_DIR)/$(SPI_FLASH_LIB) $(WEAR_LEVELLING_HOST_DIR)/$(WEAR_LEVELLING_LIB) partition_table.bin
 	g++ $(LDFLAGS) -o $(TEST_PROGRAM) $(TEST_OBJ_FILES) -L$(abspath .) -l:$(COMPONENT_LIB) -L$(SPI_FLASH_SIM_DIR) -l:$(SPI_FLASH_LIB) -L$(WEAR_LEVELLING_HOST_DIR) -l:$(WEAR_LEVELLING_LIB) -g -m32

components/partition_table/Makefile.projbuild

@@ -57,7 +57,7 @@ all_binaries: $(PARTITION_TABLE_BIN) partition_table_get_info
 
 partition_table_get_info: $(PARTITION_TABLE_BIN)
 	$(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype phy --offset $(PARTITION_TABLE_BIN)))
-	$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --type app --subtype factory --offset $(PARTITION_TABLE_BIN)))
+	$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --default-boot-partition --offset $(PARTITION_TABLE_BIN)))
 
 export APP_OFFSET
 export PHY_DATA_OFFSET

components/partition_table/gen_esp32part.py

@@ -33,7 +33,32 @@ MAX_PARTITION_LENGTH = 0xC00   # 3K for partition data (96 entries) leaves 1K in
 MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum
 PARTITION_TABLE_SIZE  = 0x1000  # Size of partition table
 
-__version__ = '1.1'
+__version__ = '1.2'
+
+APP_TYPE = 0x00
+DATA_TYPE = 0x01
+
+TYPES = {
+    "app" : APP_TYPE,
+    "data" : DATA_TYPE,
+}
+
+# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h
+SUBTYPES = {
+    APP_TYPE : {
+        "factory" : 0x00,
+        "test" : 0x20,
+    },
+    DATA_TYPE : {
+        "ota" : 0x00,
+        "phy" : 0x01,
+        "nvs" : 0x02,
+        "coredump" : 0x03,
+        "esphttpd" : 0x80,
+        "fat" : 0x81,
+        "spiffs" : 0x82,
+    },
+}
 
 quiet = False
 md5sum = True
@@ -46,9 +71,8 @@ def status(msg):
 
 def critical(msg):
     """ Print critical message to stderr """
-    if not quiet:
-        sys.stderr.write(msg)
-        sys.stderr.write('\n')
+    sys.stderr.write(msg)
+    sys.stderr.write('\n')
 
 class PartitionTable(list):
     def __init__(self):
@@ -85,7 +109,7 @@ class PartitionTable(list):
                 critical("WARNING: 0x%x address in the partition table is below 0x%x" % (e.offset, last_end))
                 e.offset = None
             if e.offset is None:
-                pad_to = 0x10000 if e.type == PartitionDefinition.APP_TYPE else 4
+                pad_to = 0x10000 if e.type == APP_TYPE else 4
                 if last_end % pad_to != 0:
                     last_end += pad_to - (last_end % pad_to)
                 e.offset = last_end
@@ -106,6 +130,36 @@ class PartitionTable(list):
         else:
             return super(PartitionTable, self).__getitem__(item)
 
+    def find_by_type(self, ptype, subtype):
+        """ Return a partition by type & subtype, returns
+        None if not found """
+        # convert ptype & subtypes names (if supplied this way) to integer values
+        try:
+            ptype = TYPES[ptype]
+        except KeyError:
+            try:
+                ptypes = int(ptype, 0)
+            except TypeError:
+                pass
+        try:
+            subtype = SUBTYPES[int(ptype)][subtype]
+        except KeyError:
+            try:
+                ptypes = int(ptype, 0)
+            except TypeError:
+                pass
+
+        for p in self:
+            if p.type == ptype and p.subtype == subtype:
+                return p
+        return None
+
+    def find_by_name(self, name):
+        for p in self:
+            if p.name == name:
+                return p
+        return None
+
     def verify(self):
         # verify each partition individually
         for p in self:
@@ -165,30 +219,6 @@ class PartitionTable(list):
         return "\n".join(rows) + "\n"
 
 class PartitionDefinition(object):
-    APP_TYPE = 0x00
-    DATA_TYPE = 0x01
-    TYPES = {
-        "app" : APP_TYPE,
-        "data" : DATA_TYPE,
-    }
-
-    # Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h 
-    SUBTYPES = {
-        APP_TYPE : {
-            "factory" : 0x00,
-            "test" : 0x20,
-            },
-        DATA_TYPE : {
-            "ota" : 0x00,
-            "phy" : 0x01,
-            "nvs" : 0x02,
-            "coredump" : 0x03,
-            "esphttpd" : 0x80,
-            "fat" : 0x81,
-            "spiffs" : 0x82,
-            },
-    }
-
     MAGIC_BYTES = b"\xAA\x50"
 
     ALIGNMENT = {
@@ -202,7 +232,7 @@ class PartitionDefinition(object):
         "encrypted" : 0
     }
 
-    # add subtypes for the 16 OTA slot values ("ota_XXX, etc.")
+    # add subtypes for the 16 OTA slot values ("ota_XX, etc.")
     for ota_slot in range(16):
         SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot
 
@@ -270,12 +300,12 @@ class PartitionDefinition(object):
     def parse_type(self, strval):
         if strval == "":
             raise InputError("Field 'type' can't be left empty.")
-        return parse_int(strval, self.TYPES)
+        return parse_int(strval, TYPES)
 
     def parse_subtype(self, strval):
         if strval == "":
             return 0 # default
-        return parse_int(strval, self.SUBTYPES.get(self.type, {}))
+        return parse_int(strval, SUBTYPES.get(self.type, {}))
 
     def parse_address(self, strval):
         if strval == "":
@@ -295,6 +325,14 @@ class PartitionDefinition(object):
         if self.size is None:
             raise ValidationError(self, "Size field is not set")
 
+        if self.name in TYPES and TYPES.get(self.name, "") != self.type:
+            critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's type (0x%x). Mistake in partition table?" % (self.name, self.type))
+        all_subtype_names = []
+        for names in (t.keys() for t in SUBTYPES.values()):
+            all_subtype_names += names
+        if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype:
+            critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype))
+
     STRUCT_FORMAT = "<2sBBLL16sL"
 
     @classmethod
@@ -348,8 +386,8 @@ class PartitionDefinition(object):
             return ":".join(self.get_flags_list())
 
         return ",".join([ self.name,
-                          lookup_keyword(self.type, self.TYPES),
-                          lookup_keyword(self.subtype, self.SUBTYPES.get(self.type, {})),
+                          lookup_keyword(self.type, TYPES),
+                          lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})),
                           addr_format(self.offset, False),
                           addr_format(self.size, True),
                           generate_text_flags()])
@@ -381,14 +419,14 @@ def main():
     parser.add_argument('--flash-size', help='Optional flash size limit, checks partition table fits in flash',
                         nargs='?', choices=[ '1MB', '2MB', '4MB', '8MB', '16MB' ])
     parser.add_argument('--disable-md5sum', help='Disable md5 checksum for the partition table', default=False, action='store_true')
-    parser.add_argument('--verify', '-v', help='Verify partition table fields', default=True, action='store_false')
-    parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
+    parser.add_argument('--no-verify', help="Don't verify partition table fields", action='store_true')
+    parser.add_argument('--verify', '-v', help="Verify partition table fields (deprecated, this behaviour is enabled by default and this flag does nothing.", action='store_true')
+    parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true')
     parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000')
     
-    parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('rb'), default=sys.stdin)
-    parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted, unless the --display argument is also passed (in which case only the summary is printed.)',
-                        nargs='?',
-                        default='-')
+    parser.add_argument('input', help='Path to CSV or binary file to parse.', type=argparse.FileType('rb'))
+    parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.',
+                        nargs='?', default='-')
 
     args = parser.parse_args()
 
@@ -405,7 +443,7 @@ def main():
         status("Parsing CSV input...")
         table = PartitionTable.from_csv(input)
 
-    if args.verify:
+    if not args.no_verify:
         status("Verifying table...")
         table.verify()
 
@@ -423,7 +461,11 @@ def main():
             f.write(output)
     else:
         output = table.to_binary()
-        with sys.stdout.buffer if args.output == '-' else open(args.output, 'wb') as f:
+        try:
+            stdout_binary = sys.stdout.buffer  # Python 3
+        except AttributeError:
+            stdout_binary = sys.stdout
+        with stdout_binary if args.output == '-' else open(args.output, 'wb') as f:
             f.write(output)
 
 

components/partition_table/partitions_two_ota.csv

@@ -3,6 +3,6 @@
 nvs,      data, nvs,     0x9000,  0x4000
 otadata,  data, ota,     0xd000,  0x2000
 phy_init, data, phy,     0xf000,  0x1000
-factory,  0,    0,       0x10000,  1M
-ota_0,    0,    ota_0,   ,         1M
-ota_1,    0,    ota_1,   ,         1M
+factory,  app,  factory, 0x10000,  1M
+ota_0,    app,  ota_0,   ,         1M
+ota_1,    app,  ota_1,   ,         1M

components/partition_table/parttool.py

@@ -48,17 +48,30 @@ def main():
     parser = argparse.ArgumentParser(description='Returns info about the required partition.')
 
     parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
-    parser.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None)
-    parser.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None)
+
+    search_type = parser.add_mutually_exclusive_group()
+    search_type.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None)
+    search_type.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None)
+    search_type.add_argument('--default-boot-partition', help='Select the default boot partition, '+
+                             'using the same fallback logic as the IDF bootloader', action="store_true")
+
     parser.add_argument('--subtype', '-s', help='The subtype of the required partition', type=str, default=None)
-    
-    parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true", default=None)
-    parser.add_argument('--size', help='Return size of required partition', action="store_true", default=None)
-    
-    parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('rb'), default=sys.stdin)
+
+    parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true")
+    parser.add_argument('--size', help='Return size of required partition', action="store_true")
+
+    parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.',
+                        type=argparse.FileType('rb'), default=sys.stdin)
 
     args = parser.parse_args()
 
+    if args.type is not None and args.subtype is None:
+        status("If --type is specified, --subtype is required")
+        return 2
+    if args.type is None and args.subtype is not None:
+        status("--subtype is only used with --type")
+        return 2
+
     quiet = args.quiet
 
     input = args.input.read()
@@ -71,36 +84,30 @@ def main():
         status("Parsing CSV input...")
         table = gen.PartitionTable.from_csv(input)
 
-    if args.partition_name is not None:
-        offset = 0
-        size = 0
-        for p in table:
-            if p.name == args.partition_name:
-                offset = p.offset
-                size = p.size
-                break;
-        if args.offset is not None:
-            print('0x%x ' % (offset))
-        if args.size is not None:
-            print('0x%x' % (size))
-        return 0
-    
-    if args.type is not None and args.subtype is not None:
-        offset = 0
-        size = 0
-        TYPES = gen.PartitionDefinition.TYPES
-        SUBTYPES = gen.PartitionDefinition.SUBTYPES
-        for p in table:
-            if p.type == TYPES[args.type]:
-                if p.subtype == SUBTYPES[TYPES[args.type]][args.subtype]:
-                    offset = p.offset
-                    size = p.size
-                    break;
-        if args.offset is not None:
-            print('0x%x ' % (offset))
-        if args.size is not None:
-            print('0x%x' % (size))
-        return 0
+    found_partition = None
+
+    if args.default_boot_partition:
+        search = [ "factory" ] + [ "ota_%d" % d for d in range(16) ]
+        for subtype in search:
+            found_partition = table.find_by_type("app", subtype)
+            if found_partition is not None:
+                break
+    elif args.partition_name is not None:
+        found_partition = table.find_by_name(args.partition_name)
+    elif args.type is not None:
+        found_partition = table.find_by_type(args.type, args.subtype)
+    else:
+        raise RuntimeError("invalid partition selection choice")
+
+    if found_partition is None:
+        return 1  # nothing found
+
+    if args.offset:
+        print('0x%x ' % (found_partition.offset))
+    if args.size:
+        print('0x%x' % (found_partition.size))
+
+    return 0
 
 class InputError(RuntimeError):
     def __init__(self, e):
@@ -115,7 +122,8 @@ class ValidationError(InputError):
 
 if __name__ == '__main__':
     try:
-        main()
+        r = main()
+        sys.exit(r)
     except InputError as e:
         print(e, file=sys.stderr)
         sys.exit(2)

components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py

@@ -7,6 +7,7 @@ import sys
 import subprocess
 import tempfile
 import os
+import StringIO
 sys.path.append("..")
 from gen_esp32part import *
 
@@ -236,10 +237,10 @@ class BinaryParserTests(unittest.TestCase):
         t.verify()
 
         self.assertEqual(3, len(t))
-        self.assertEqual(t[0].type, PartitionDefinition.APP_TYPE)
+        self.assertEqual(t[0].type, APP_TYPE)
         self.assertEqual(t[0].name, "factory")
 
-        self.assertEqual(t[1].type, PartitionDefinition.DATA_TYPE)
+        self.assertEqual(t[1].type, DATA_TYPE)
         self.assertEqual(t[1].name, "data")
 
         self.assertEqual(t[2].type, 0x10)
@@ -319,16 +320,18 @@ class CommandLineTests(unittest.TestCase):
                 f.write(LONGER_BINARY_TABLE)
 
             # run gen_esp32part.py to convert binary file to CSV
-            subprocess.check_call([sys.executable, "../gen_esp32part.py",
-                                   binpath, csvpath])
+            output = subprocess.check_output([sys.executable, "../gen_esp32part.py",
+                                   binpath, csvpath], stderr=subprocess.STDOUT)
             # reopen the CSV and check the generated binary is identical
+            self.assertNotIn("WARNING", output)
             with open(csvpath, 'r') as f:
                 from_csv = PartitionTable.from_csv(f.read())
             self.assertEqual(_strip_trailing_ffs(from_csv.to_binary()), LONGER_BINARY_TABLE)
 
             # run gen_esp32part.py to conver the CSV to binary again
-            subprocess.check_call([sys.executable, "../gen_esp32part.py",
-                                   csvpath, binpath])
+            output = subprocess.check_output([sys.executable, "../gen_esp32part.py",
+                                              csvpath, binpath], stderr=subprocess.STDOUT)
+            self.assertNotIn("WARNING", output)
             # assert that file reads back as identical
             with open(binpath, 'rb') as f:
                 binary_readback = f.read()
@@ -356,5 +359,75 @@ app,app, factory, 32K, 1M
             t.verify()
 
 
+    def test_warnings(self):
+        try:
+            sys.stderr = StringIO.StringIO()  # capture stderr
+
+            csv_1 = "app, 1, 2, 32K, 1M\n"
+            PartitionTable.from_csv(csv_1).verify()
+            self.assertIn("WARNING", sys.stderr.getvalue())
+            self.assertIn("partition type", sys.stderr.getvalue())
+
+            sys.stderr = StringIO.StringIO()
+            csv_2 = "ota_0, app, ota_1, , 1M\n"
+            PartitionTable.from_csv(csv_2).verify()
+            self.assertIn("WARNING", sys.stderr.getvalue())
+            self.assertIn("partition subtype", sys.stderr.getvalue())
+
+        finally:
+            sys.stderr = sys.__stderr__
+
+class PartToolTests(unittest.TestCase):
+
+    def _run_parttool(self, csvcontents, args):
+        csvpath = tempfile.mktemp()
+        with open(csvpath, "w") as f:
+            f.write(csvcontents)
+        try:
+            output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") + [ csvpath ],
+                                           stderr=subprocess.STDOUT)
+            self.assertNotIn("WARNING", output)
+            m = re.search("0x[0-9a-fA-F]+", output)
+            return m.group(0) if m else ""
+        finally:
+            os.remove(csvpath)
+
+    def test_find_basic(self):
+        csv = """
+nvs,      data, nvs,     0x9000,  0x4000
+otadata,  data, ota,     0xd000,  0x2000
+phy_init, data, phy,     0xf000,  0x1000
+factory,  app, factory, 0x10000,  1M
+        """
+        rpt = lambda args: self._run_parttool(csv, args)
+
+        self.assertEqual(
+            rpt("--type data --subtype nvs --offset"), "0x9000")
+        self.assertEqual(
+            rpt("--type data --subtype nvs --size"), "0x4000")
+        self.assertEqual(
+            rpt("--partition-name otadata --offset"), "0xd000")
+        self.assertEqual(
+            rpt("--default-boot-partition --offset"), "0x10000")
+
+    def test_fallback(self):
+        csv = """
+nvs,      data, nvs,     0x9000,  0x4000
+otadata,  data, ota,     0xd000,  0x2000
+phy_init, data, phy,     0xf000,  0x1000
+ota_0,  app,    ota_0,   0x30000,  1M
+ota_1,  app,    ota_1,          ,  1M
+        """
+        rpt = lambda args: self._run_parttool(csv, args)
+
+        self.assertEqual(
+            rpt("--type app --subtype ota_1 --offset"), "0x130000")
+        self.assertEqual(
+            rpt("--default-boot-partition --offset"), "0x30000")  # ota_0
+        csv_mod = csv.replace("ota_0", "ota_2")
+        self.assertEqual(
+            self._run_parttool(csv_mod, "--default-boot-partition --offset"),
+            "0x130000")  # now default is ota_1
+
 if __name__ =="__main__":
     unittest.main()

components/spiffs/test_spiffs_host/Makefile

@@ -67,7 +67,7 @@ $(COMPONENT_LIB): $(OBJ_FILES)
 lib: $(COMPONENT_LIB)
 
 partitions_table.bin: partitions_table.csv
-	python ../../partition_table/gen_esp32part.py --verify $< $@
+	python ../../partition_table/gen_esp32part.py $< $@
 
 $(TEST_PROGRAM): lib $(TEST_OBJ_FILES) $(SPI_FLASH_SIM_DIR)/$(SPI_FLASH_LIB) partitions_table.bin
 	g++ $(LDFLAGS) -o $(TEST_PROGRAM) $(TEST_OBJ_FILES) -L$(abspath .) -l:$(COMPONENT_LIB) -L$(SPI_FLASH_SIM_DIR) -l:$(SPI_FLASH_LIB) -g -m32

components/wear_levelling/test_wl_host/Makefile

@@ -66,7 +66,7 @@ force:
 lib: $(COMPONENT_LIB)
 
 partition_table.bin: partition_table.csv
-	python ../../../components/partition_table/gen_esp32part.py --verify $< $@
+	python ../../../components/partition_table/gen_esp32part.py $< $@
 
 $(TEST_PROGRAM): lib $(TEST_OBJ_FILES) $(SPI_FLASH_SIM_DIR)/$(SPI_FLASH_LIB) partition_table.bin
 	g++ $(LDFLAGS) -o $@  $(TEST_OBJ_FILES) -L$(abspath .) -l:$(COMPONENT_LIB) -L$(SPI_FLASH_SIM_DIR) -l:$(SPI_FLASH_LIB) -g -m32

docs/en/api-guides/partition-tables.rst

@@ -136,18 +136,16 @@ If you configure the partition table CSV name in ``make menuconfig`` and then ``
 
 To convert CSV to Binary manually::
 
-  python gen_esp32part.py --verify input_partitions.csv binary_partitions.bin
+  python gen_esp32part.py input_partitions.csv binary_partitions.bin
 
 To convert binary format back to CSV::
 
-  python gen_esp32part.py --verify binary_partitions.bin input_partitions.csv
+  python gen_esp32part.py binary_partitions.bin input_partitions.csv
 
 To display the contents of a binary partition table on stdout (this is how the summaries displayed when running `make partition_table` are generated::
 
   python gen_esp32part.py binary_partitions.bin
 
-``gen_esp32part.py`` takes one optional argument, ``--verify``, which will also verify the partition table during conversion (checking for overlapping partitions, unaligned partitions, etc.)
-
 MD5 checksum
 ~~~~~~~~~~~~