heap: Add integer overflow checks

components/esp32/ld/esp32.rom.ld

@@ -50,7 +50,6 @@ PROVIDE ( cache_sram_mmu_set_rom = 0x400097f4 );
 /* This is static function, but can be used, not generated by script*/
 PROVIDE ( calc_rtc_memory_crc = 0x40008170 );
 PROVIDE ( calloc = 0x4000bee4 );
-PROVIDE ( _calloc_r = 0x4000bbf8 );
 PROVIDE ( __clear_cache = 0x40063860 );
 PROVIDE ( _close_r = 0x4000bd3c );
 PROVIDE ( __clrsbdi2 = 0x40064a38 );

components/heap/heap_caps.c

@@ -305,12 +305,18 @@ IRAM_ATTR void *heap_caps_realloc( void *ptr, size_t size, int caps)
 
 IRAM_ATTR void *heap_caps_calloc( size_t n, size_t size, uint32_t caps)
 {
-    void *r;
-    r = heap_caps_malloc(n*size, caps);
-    if (r != NULL) {
-        bzero(r, n*size);
+    void *result;
+    size_t size_bytes;
+
+    if (__builtin_mul_overflow(n, size, &size_bytes)) {
+        return NULL;
     }
-    return r;
+
+    result = heap_caps_malloc(size_bytes, caps);
+    if (result != NULL) {
+        bzero(result, size_bytes);
+    }
+    return result;
 }
 
 size_t heap_caps_get_free_size( uint32_t caps )

components/heap/multi_heap_poisoning.c

@@ -175,6 +175,9 @@ static bool verify_fill_pattern(void *data, size_t size, bool print_errors, bool
 
 void *multi_heap_malloc(multi_heap_handle_t heap, size_t size)
 {
+    if(size > SIZE_MAX - POISON_OVERHEAD) {
+        return NULL;
+    }
     multi_heap_internal_lock(heap);
     poison_head_t *head = multi_heap_malloc_impl(heap, size + POISON_OVERHEAD);
     uint8_t *data = NULL;
@@ -216,6 +219,9 @@ void *multi_heap_realloc(multi_heap_handle_t heap, void *p, size_t size)
     poison_head_t *new_head;
     void *result = NULL;
 
+    if(size > SIZE_MAX - POISON_OVERHEAD) {
+        return NULL;
+    }
     if (p == NULL) {
         return multi_heap_malloc(heap, size);
     }

components/heap/test/test_malloc.c

@@ -65,7 +65,6 @@ TEST_CASE("Malloc/overwrite, then free all available DRAM", "[heap]")
     TEST_ASSERT(m1==m2);
 }
 
-
 #if CONFIG_SPIRAM_USE_MALLOC
 
 #if (CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL > 1024)
@@ -87,4 +86,25 @@ TEST_CASE("Check if reserved DMA pool still can allocate even when malloc()'ed m
 }
 #endif
 
-#endif
+#endif
+
+TEST_CASE("alloc overflows should all fail", "[heap]")
+{
+    /* allocates 8 bytes */
+    TEST_ASSERT_NULL(calloc(SIZE_MAX / 2 + 4, 2));
+
+    /* will overflow if any poisoning is enabled
+       (should fail for sensible OOM reasons, otherwise) */
+    TEST_ASSERT_NULL(malloc(SIZE_MAX - 1));
+    TEST_ASSERT_NULL(calloc(SIZE_MAX - 1, 1));
+}
+
+TEST_CASE("unreasonable allocs should all fail", "[heap]")
+{
+    TEST_ASSERT_NULL(calloc(16, 1024*1024));
+    TEST_ASSERT_NULL(malloc(16*1024*1024));
+    TEST_ASSERT_NULL(malloc(SIZE_MAX / 2));
+    TEST_ASSERT_NULL(malloc(SIZE_MAX - 256));
+    TEST_ASSERT_NULL(malloc(xPortGetFreeHeapSize() - 1));
+}
+

components/newlib/syscalls.c

@@ -47,11 +47,17 @@ void* IRAM_ATTR _realloc_r(struct _reent *r, void* ptr, size_t size)
     return heap_caps_realloc_default( ptr, size );
 }
 
-void* IRAM_ATTR _calloc_r(struct _reent *r, size_t count, size_t size)
+void* IRAM_ATTR _calloc_r(struct _reent *r, size_t nmemb, size_t size)
 {
-    void* result = heap_caps_malloc_default(count * size);
-    if (result) {
-        bzero(result, count * size);
+    void *result;
+    size_t size_bytes;
+    if (__builtin_mul_overflow(nmemb, size, &size_bytes)) {
+        return NULL;
+    }
+
+    result = malloc(size_bytes);
+    if (result != NULL) {
+        bzero(result, size_bytes);
     }
     return result;
 }