Merge branch 'feature/malloc_psram' into 'master'

Add logic to make external RAM usable with malloc()



See merge request !1278

components/esp32/Kconfig

@@ -47,7 +47,7 @@ config SPIRAM_BOOT_INIT
 
 choice SPIRAM_USE
     prompt "SPI RAM access method"
-    default SPIRAM_USE_MEMMAP
+    default SPIRAM_USE_MALLOC
     help
         The SPI RAM can be accessed in multiple methods: by just having it available as an unmanaged
         memory region in the ESP32 memory map, by integrating it in the ESP32s heap as 'special' memory
@@ -59,8 +59,7 @@ config SPIRAM_USE_MEMMAP
 config SPIRAM_USE_CAPS_ALLOC
     bool "Make RAM allocatable using heap_caps_malloc(..., MALLOC_CAP_SPIRAM)"
 config SPIRAM_USE_MALLOC
-    bool "Make RAM allocatable using malloc as well"
-    depends on TO_BE_DONE
+    bool "Make RAM allocatable using malloc() as well"
 endchoice
 
 choice SPIRAM_TYPE
@@ -118,6 +117,46 @@ config SPIRAM_CACHE_WORKAROUND
         with the workaround and located in flash instead.
 
 
+config SPIRAM_MALLOC_ALWAYSINTERNAL
+    int "Maximum malloc() size, in bytes, to always put in internal memory"
+    depends on SPIRAM_USE_MALLOC
+    default 16384
+    range 0 131072
+    help
+        If malloc() is capable of also allocating SPI-connected ram, its allocation strategy will prefer to allocate chunks less
+        than this size in internal memory, while allocations larger than this will be done from external RAM.
+        If allocation from the preferred region fails, an attempt is made to allocate from the non-preferred
+        region instead, so malloc() will not suddenly fail when either internal or external memory is full.
+
+config SPIRAM_MALLOC_RESERVE_INTERNAL
+    int "Reserve this amount of bytes for data that specifically needs to be in DMA or internal memory"
+    depends on SPIRAM_USE_MALLOC
+    default 32768
+    range 0 131072
+    help
+        Because the external/internal RAM allocation strategy is not always perfect, it sometimes may happen
+        that the internal memory is entirely filled up. This causes allocations that are specifically done in
+        internal memory, for example the stack for new tasks or memory to service DMA or have memory that's 
+        also available when SPI cache is down, to fail. This option reserves a pool specifically for requests 
+        like that; the memory in this pool is not given out when a normal malloc() is called.
+        
+        Set this to 0 to disable this feature.
+        
+        Note that because FreeRTOS stacks are forced to internal memory, they will also use this memory pool;
+        be sure to keep this in mind when adjusting this value.
+
+config SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY
+    bool "Allow external memory as an argument to xTaskCreateStatic"
+    default n
+    depends on SPIRAM_USE_MALLOC
+    help
+        Because some bits of the ESP32 code environment cannot be recompiled with the cache workaround, normally
+        tasks cannot be safely run with their stack residing in external memory; for this reason xTaskCreate and
+        friends always allocate stack in internal memory and xTaskCreateStatic will check if the memory passed
+        to it is in internal memory. If you have a task that needs a large amount of stack and does not call on
+        ROM code in any way (no direct calls, but also no Bluetooth/WiFi), you can try to disable this and use
+        xTaskCreateStatic to create the tasks stack in external memory.
+
 endmenu
 
 config MEMMAP_TRACEMEM
@@ -776,6 +815,7 @@ config ESP32_WIFI_STATIC_TX_BUFFER
     bool "STATIC"
 config ESP32_WIFI_DYNAMIC_TX_BUFFER
     bool "DYNAMIC"
+    depends on !SPIRAM_USE_MALLOC
 endchoice
 
 config ESP32_WIFI_TX_BUFFER_TYPE

components/esp32/cpu_start.c

@@ -262,6 +262,16 @@ void start_cpu0_default(void)
         ESP_EARLY_LOGE(TAG, "External RAM could not be added to heap!");
         abort();
     }
+#if CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL
+    r=esp_spiram_reserve_dma_pool(CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL);
+    if (r != ESP_OK) {
+        ESP_EARLY_LOGE(TAG, "Could not reserve internal/DMA pool!");
+        abort();
+    }
+#endif
+#if CONFIG_SPIRAM_USE_MALLOC
+    heap_caps_malloc_extmem_enable(CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL);
+#endif
 #endif
 
 //Enable trace memory and immediately start trace.

components/esp32/include/esp_spiram.h

@@ -64,4 +64,16 @@ void esp_spiram_writeback_cache();
 
 
 
+/**
+ * @brief Reserve a pool of internal memory for specific DMA/internal allocations
+ *
+ * @param size Size of reserved pool in bytes
+ *
+ * @return
+ *          - ESP_OK on success
+ *          - ESP_ERR_NO_MEM when no memory available for pool
+ */
+esp_err_t esp_spiram_reserve_dma_pool(size_t size);
+
+
 #endif

components/esp32/intr_alloc.c

@@ -222,7 +222,7 @@ static vector_desc_t *get_desc_for_int(int intno, int cpu)
 {
     vector_desc_t *vd=find_desc_for_int(intno, cpu);
     if (vd==NULL) {
-        vector_desc_t *newvd=malloc(sizeof(vector_desc_t));
+        vector_desc_t *newvd=heap_caps_malloc(sizeof(vector_desc_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
         if (newvd==NULL) return NULL;
         memset(newvd, 0, sizeof(vector_desc_t));
         newvd->intno=intno;
@@ -574,7 +574,7 @@ esp_err_t esp_intr_alloc_intrstatus(int source, int flags, uint32_t intrstatusre
     if (source==ETS_INTERNAL_PROFILING_INTR_SOURCE) force=ETS_INTERNAL_PROFILING_INTR_NO;
 
     //Allocate a return handle. If we end up not needing it, we'll free it later on.
-    ret=malloc(sizeof(intr_handle_data_t));
+    ret=heap_caps_malloc(sizeof(intr_handle_data_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
     if (ret==NULL) return ESP_ERR_NO_MEM;
 
     portENTER_CRITICAL(&spinlock);

components/esp32/spiram.c

@@ -124,11 +124,24 @@ esp_err_t esp_spiram_init()
 
 esp_err_t esp_spiram_add_to_heapalloc()
 {
+    ESP_EARLY_LOGI(TAG, "Adding pool of %dK of external SPI memory to heap allocator", CONFIG_SPIRAM_SIZE/1024);
     //Add entire external RAM region to heap allocator. Heap allocator knows the capabilities of this type of memory, so there's
     //no need to explicitly specify them.
     return heap_caps_add_region((intptr_t)SOC_EXTRAM_DATA_LOW, (intptr_t)SOC_EXTRAM_DATA_LOW + CONFIG_SPIRAM_SIZE-1);
 }
 
+
+static uint8_t *dma_heap;
+
+esp_err_t esp_spiram_reserve_dma_pool(size_t size) {
+    if (size==0) return ESP_OK; //no-op
+    ESP_EARLY_LOGI(TAG, "Reserving pool of %dK of internal memory for DMA/internal allocations", size/1024);
+    dma_heap=heap_caps_malloc(size, MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
+    if (!dma_heap) return ESP_ERR_NO_MEM;
+    uint32_t caps[]={MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL, 0, MALLOC_CAP_8BIT|MALLOC_CAP_32BIT};
+    return heap_caps_add_region_with_caps(caps, dma_heap, dma_heap+size-1);
+}
+
 size_t esp_spiram_get_size()
 {
     return CONFIG_SPIRAM_SIZE;

components/esp32/system_api.c

@@ -360,12 +360,12 @@ void system_restore(void)
 
 uint32_t esp_get_free_heap_size( void )
 {
-    return heap_caps_get_free_size( MALLOC_CAP_8BIT );
+    return heap_caps_get_free_size( MALLOC_CAP_DEFAULT );
 }
 
 uint32_t esp_get_minimum_free_heap_size( void )
 {
-    return heap_caps_get_minimum_free_size( MALLOC_CAP_8BIT );
+    return heap_caps_get_minimum_free_size( MALLOC_CAP_DEFAULT );
 }
 
 uint32_t system_get_free_heap_size(void) __attribute__((alias("esp_get_free_heap_size")));

components/freertos/include/freertos/portmacro.h

@@ -82,6 +82,9 @@ extern "C" {
 #include "esp_crosscore_int.h"
 
 
+#include <esp_heap_caps.h>
+#include "soc/soc_memory_layout.h"
+
 //#include "xtensa_context.h"
 
 /*-----------------------------------------------------------
@@ -245,6 +248,18 @@ static inline unsigned portENTER_CRITICAL_NESTED() { unsigned state = XTOS_SET_I
 #define portSET_INTERRUPT_MASK_FROM_ISR()            portENTER_CRITICAL_NESTED()
 #define portCLEAR_INTERRUPT_MASK_FROM_ISR(state)     portEXIT_CRITICAL_NESTED(state)
 
+//Because the ROM routines don't necessarily handle a stack in external RAM correctly, we force
+//the stack memory to always be internal.
+#define pvPortMallocTcbMem(size) heap_caps_malloc(size, MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT)
+#define pvPortMallocStackMem(size)  heap_caps_malloc(size, MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT)
+
+//xTaskCreateStatic uses these functions to check incoming memory.
+#define portVALID_TCB_MEM(ptr) (esp_ptr_internal(ptr) && esp_ptr_byte_accessible(ptr))
+#ifndef CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY
+#define portVALID_STACK_MEM(ptr) esp_ptr_byte_accessible(ptr)
+#else
+#define portVALID_STACK_MEM(ptr) (esp_ptr_internal(ptr) && esp_ptr_byte_accessible(ptr))
+#endif
 
 /*
  * Wrapper for the Xtensa compare-and-set instruction. This subroutine will atomically compare

components/freertos/tasks.c

@@ -677,8 +677,8 @@ void taskYIELD_OTHER_CORE( BaseType_t xCoreID, UBaseType_t uxPriority )
 	TCB_t *pxNewTCB;
 	TaskHandle_t xReturn;
 
-		configASSERT( puxStackBuffer != NULL );
-		configASSERT( pxTaskBuffer != NULL );
+		configASSERT( portVALID_TCB_MEM(pxTaskBuffer) );
+		configASSERT( portVALID_STACK_MEM(puxStackBuffer) );
 		configASSERT( (xCoreID>=0 && xCoreID<portNUM_PROCESSORS) || (xCoreID==tskNO_AFFINITY) );
 
 		if( ( pxTaskBuffer != NULL ) && ( puxStackBuffer != NULL ) )
@@ -724,7 +724,7 @@ void taskYIELD_OTHER_CORE( BaseType_t xCoreID, UBaseType_t uxPriority )
 			/* Allocate space for the TCB.  Where the memory comes from depends
 			on the implementation of the port malloc function and whether or
 			not static allocation is being used. */
-			pxNewTCB = ( TCB_t * ) pvPortMalloc( sizeof( TCB_t ) );
+			pxNewTCB = ( TCB_t * ) pvPortMallocTcbMem( sizeof( TCB_t ) );
 
 			if( pxNewTCB != NULL )
 			{
@@ -777,14 +777,14 @@ void taskYIELD_OTHER_CORE( BaseType_t xCoreID, UBaseType_t uxPriority )
 			/* Allocate space for the TCB.  Where the memory comes from depends on
 			the implementation of the port malloc function and whether or not static
 			allocation is being used. */
-			pxNewTCB = ( TCB_t * ) pvPortMalloc( sizeof( TCB_t ) );
+			pxNewTCB = ( TCB_t * ) pvPortMallocTcbMem( sizeof( TCB_t ) );
 
 			if( pxNewTCB != NULL )
 			{
 				/* Allocate space for the stack used by the task being created.
 				The base of the stack memory stored in the TCB so the task can
 				be deleted later if required. */
-				pxNewTCB->pxStack = ( StackType_t * ) pvPortMalloc( ( ( ( size_t ) usStackDepth ) * sizeof( StackType_t ) ) ); /*lint !e961 MISRA exception as the casts are only redundant for some ports. */
+				pxNewTCB->pxStack = ( StackType_t * ) pvPortMallocStackMem( ( ( ( size_t ) usStackDepth ) * sizeof( StackType_t ) ) ); /*lint !e961 MISRA exception as the casts are only redundant for some ports. */
 
 				if( pxNewTCB->pxStack == NULL )
 				{
@@ -799,12 +799,12 @@ void taskYIELD_OTHER_CORE( BaseType_t xCoreID, UBaseType_t uxPriority )
 		StackType_t *pxStack;
 
 			/* Allocate space for the stack used by the task being created. */
-			pxStack = ( StackType_t * ) pvPortMalloc( ( ( ( size_t ) usStackDepth ) * sizeof( StackType_t ) ) ); /*lint !e961 MISRA exception as the casts are only redundant for some ports. */
+			pxStack = ( StackType_t * ) pvPortMallocStackMem( ( ( ( size_t ) usStackDepth ) * sizeof( StackType_t ) ) ); /*lint !e961 MISRA exception as the casts are only redundant for some ports. */
 
 			if( pxStack != NULL )
 			{
 				/* Allocate space for the TCB. */
-				pxNewTCB = ( TCB_t * ) pvPortMalloc( sizeof( TCB_t ) ); /*lint !e961 MISRA exception as the casts are only redundant for some paths. */
+				pxNewTCB = ( TCB_t * ) pvPortMallocTcbMem( sizeof( TCB_t ) ); /*lint !e961 MISRA exception as the casts are only redundant for some paths. */
 
 				if( pxNewTCB != NULL )
 				{

components/heap/heap_caps.c

@@ -133,12 +133,36 @@ IRAM_ATTR void *heap_caps_malloc( size_t size, uint32_t caps )
     return NULL;
 }
 
+
+#define MALLOC_DISABLE_EXTERNAL_ALLOCS -1
+//Dual-use: -1 (=MALLOC_DISABLE_EXTERNAL_ALLOCS) disables allocations in external memory, >=0 sets the limit for allocations preferring internal memory.
+static int malloc_alwaysinternal_limit=MALLOC_DISABLE_EXTERNAL_ALLOCS;
+
+void heap_caps_malloc_extmem_enable(size_t limit)
+{
+    malloc_alwaysinternal_limit=limit;
+}
+
 /*
  Default memory allocation implementation. Should return standard 8-bit memory. malloc() essentially resolves to this function.
 */
 IRAM_ATTR void *heap_caps_malloc_default( size_t size )
 {
-    return heap_caps_malloc( size, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL );
+    if (malloc_alwaysinternal_limit==MALLOC_DISABLE_EXTERNAL_ALLOCS) {
+        return heap_caps_malloc( size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL);
+    } else {
+        void *r;
+        if (size <= malloc_alwaysinternal_limit) {
+            r=heap_caps_malloc( size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL );
+        } else {
+            r=heap_caps_malloc( size, MALLOC_CAP_DEFAULT | MALLOC_CAP_SPIRAM );
+        }
+        if (r==NULL) {
+            //try again while being less picky
+            r=heap_caps_malloc( size, MALLOC_CAP_DEFAULT );
+        }
+        return r;
+    }
 }
 
 /*
@@ -147,7 +171,21 @@ IRAM_ATTR void *heap_caps_malloc_default( size_t size )
  */
 IRAM_ATTR void *heap_caps_realloc_default( void *ptr, size_t size )
 {
-    return heap_caps_realloc( ptr, size, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL );
+    if (malloc_alwaysinternal_limit==MALLOC_DISABLE_EXTERNAL_ALLOCS) {
+        return heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL );
+    } else {
+        void *r;
+        if (size <= malloc_alwaysinternal_limit) {
+            r=heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT | MALLOC_CAP_INTERNAL );
+        } else {
+            r=heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT | MALLOC_CAP_SPIRAM );
+        }
+        if (r==NULL && size>0) {
+            //We needed to allocate memory, but we didn't. Try again while being less picky.
+            r=heap_caps_realloc( ptr, size, MALLOC_CAP_DEFAULT );
+        }
+        return r;
+    }
 }
 
 

components/heap/heap_caps_init.c

@@ -230,6 +230,14 @@ esp_err_t heap_caps_add_region_with_caps(const uint32_t caps[], intptr_t start,
         return ESP_ERR_INVALID_ARG;
     }
 
+    //Check if region overlaps the start and/or end of an existing region. If so, the
+    //region is invalid (or maybe added twice)
+    heap_t *heap;
+    SLIST_FOREACH(heap, &registered_heaps, next) {
+        if ( start <= heap->start &&  heap->start <=end ) return ESP_FAIL;
+        if ( start <= heap->end &&  heap->end <=end ) return ESP_FAIL;
+    }
+
     heap_t *p_new = malloc(sizeof(heap_t));
     if (p_new == NULL) {
         err = ESP_ERR_NO_MEM;

components/heap/include/esp_heap_caps.h

@@ -32,6 +32,7 @@
 #define MALLOC_CAP_PID7             (1<<9)  ///< Memory must be mapped to PID7 memory space (PIDs are not currently used)
 #define MALLOC_CAP_SPIRAM           (1<<10) ///< Memory must be in SPI RAM
 #define MALLOC_CAP_INTERNAL         (1<<11) ///< Memory must be internal; specifically it should not disappear when flash/spiram cache is switched off
+#define MALLOC_CAP_DEFAULT          (1<<12) ///< Memory can be returned in a non-capability-specific memory allocation (e.g. malloc(), calloc()) call
 #define MALLOC_CAP_INVALID          (1<<31) ///< Memory can't be used / list end marker
 
 /**
@@ -172,3 +173,18 @@ void heap_caps_print_heap_info( uint32_t caps );
  * @return True if all heaps are valid, False if at least one heap is corrupt.
  */
 bool heap_caps_check_integrity(uint32_t caps, bool print_errors);
+
+
+
+/**
+ * @brief Enable malloc() in external memory and set limit below which 
+ *        malloc() attempts are placed in internal memory.
+ *
+ * When external memory is in use, the allocation strategy is to initially try to
+ * satisfy smaller allocation requests with internal memory and larger requests
+ * with external memory. This sets the limit between the two, as well as generally
+ * enabling allocation in external memory.
+ *
+ * @param limit       Limit, in bytes.
+ */
+void heap_caps_malloc_extmem_enable(size_t limit);

components/heap/include/esp_heap_caps_init.h

@@ -73,8 +73,11 @@ esp_err_t heap_caps_add_region(intptr_t start, intptr_t end);
  * @param start Start address of new region.
  * @param end End address of new region.
  *
- * @return ESP_OK on success, ESP_ERR_INVALID_ARG if a parameter is invalid, ESP_ERR_NO_MEM if no
- * memory to register new heap.
+ * @return 
+ *         - ESP_OK on success
+ *         - ESP_ERR_INVALID_ARG if a parameter is invalid
+ *         - ESP_ERR_NO_MEM if no memory to register new heap.
+ *         - ESP_FAIL if region overlaps the start and/or end of an existing region
  */
 esp_err_t heap_caps_add_region_with_caps(const uint32_t caps[], intptr_t start, intptr_t end);
 

components/heap/test/test_malloc.c

@@ -15,32 +15,42 @@
 #include "soc/uart_reg.h"
 #include "soc/dport_reg.h"
 #include "soc/io_mux_reg.h"
+#include "esp_heap_caps.h"
 
 #include "esp_panic.h"
+#include "sdkconfig.h"
+
+
+static int **allocatedMem;
+static int noAllocated;
+
 
 static int tryAllocMem() {
-    int **mem;
-    int i, noAllocated, j;
+    int i, j;
+    const int allocateMaxK=1024*5; //try to allocate a max of 5MiB
 
-    mem=malloc(sizeof(int *)*1024);
-    if (!mem) return 0;
+    allocatedMem=malloc(sizeof(int *)*allocateMaxK);
+    if (!allocatedMem) return 0;
 
-    for (i=0; i<1024; i++) {
-        mem[i]=malloc(1024);
-        if (mem[i]==NULL) break;
-        for (j=0; j<1024/4; j++) mem[i][j]=(0xdeadbeef);
+    for (i=0; i<allocateMaxK; i++) {
+        allocatedMem[i]=malloc(1024);
+        if (allocatedMem[i]==NULL) break;
+        for (j=0; j<1024/4; j++) allocatedMem[i][j]=(0xdeadbeef);
     }
-
     noAllocated=i;
+    return i;
+}
+
 
+static void tryAllocMemFree() {
+    int i, j;
     for (i=0; i<noAllocated; i++) {
         for (j=0; j<1024/4; j++) {
-            TEST_ASSERT(mem[i][j]==(0xdeadbeef));
+            TEST_ASSERT(allocatedMem[i][j]==(0xdeadbeef));
         }
-        free(mem[i]);
+        free(allocatedMem[i]);
     }
-    free(mem);
-    return noAllocated;
+    free(allocatedMem);
 }
 
 
@@ -48,8 +58,33 @@ TEST_CASE("Malloc/overwrite, then free all available DRAM", "[heap]")
 {
     int m1=0, m2=0;
     m1=tryAllocMem();
+    tryAllocMemFree();
     m2=tryAllocMem();
+    tryAllocMemFree();
     printf("Could allocate %dK on first try, %dK on 2nd try.\n", m1, m2);
     TEST_ASSERT(m1==m2);
 }
 
+
+#if CONFIG_SPIRAM_USE_MALLOC
+
+#if (CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL > 1024)
+TEST_CASE("Check if reserved DMA pool still can allocate even when malloc()'ed memory is exhausted", "[heap]")
+{
+    char** dmaMem=malloc(sizeof(char*)*512);
+    assert(dmaMem);
+    int m=tryAllocMem();
+    int i=0;
+    for (i=0; i<512; i++) {
+        dmaMem[i]=heap_caps_malloc(1024, MALLOC_CAP_DMA);
+        if (dmaMem[i]==NULL) break;
+    }
+    for (int j=0; j<i; j++) free(dmaMem[j]);
+    free(dmaMem);
+    tryAllocMemFree();
+    printf("Could allocate %dK of DMA memory after allocating all of %dK of normal memory.\n", i, m);
+    TEST_ASSERT(i);
+}
+#endif
+
+#endif

components/log/log.c

@@ -354,7 +354,7 @@ void esp_log_buffer_hex_internal(const char *tag, const void *buffer, uint16_t b
         } else {
             bytes_cur_line = buff_len;
         }
-        if ( !esp_ptr_byte_accesible(buffer) ) {
+        if ( !esp_ptr_byte_accessible(buffer) ) {
             //use memcpy to get around alignment issue
             memcpy( temp_buffer, buffer, (bytes_cur_line+3)/4*4 );
             ptr_line = temp_buffer;
@@ -386,7 +386,7 @@ void esp_log_buffer_char_internal(const char *tag, const void *buffer, uint16_t
         } else {
             bytes_cur_line = buff_len;
         }
-        if ( !esp_ptr_byte_accesible(buffer) ) {
+        if ( !esp_ptr_byte_accessible(buffer) ) {
             //use memcpy to get around alignment issue
             memcpy( temp_buffer, buffer, (bytes_cur_line+3)/4*4 );
             ptr_line = temp_buffer;
@@ -421,7 +421,7 @@ void esp_log_buffer_hexdump_internal( const char *tag, const void *buffer, uint1
         } else {
             bytes_cur_line = buff_len;
         }
-        if ( !esp_ptr_byte_accesible(buffer) ) {
+        if ( !esp_ptr_byte_accessible(buffer) ) {
             //use memcpy to get around alignment issue
             memcpy( temp_buffer, buffer, (bytes_cur_line+3)/4*4 );
             ptr_line = temp_buffer;

components/soc/esp32/include/soc/soc.h

@@ -295,10 +295,15 @@
 #define SOC_DMA_LOW  0x3FFAE000
 #define SOC_DMA_HIGH 0x40000000
 
-// Region of memory that is byte-accessible. See esp_ptr_byte_accesible().
+// Region of memory that is byte-accessible. See esp_ptr_byte_accessible().
 #define SOC_BYTE_ACCESSIBLE_LOW     0x3FFAE000
 #define SOC_BYTE_ACCESSIBLE_HIGH    0x40000000
 
+//Region of memory that is internal, as in on the same silicon die as the ESP32 CPUs (excluding RTC data region, that's checked separately.) See esp_ptr_internal().
+#define SOC_MEM_INTERNAL_LOW        0x3F400000
+#define SOC_MEM_INTERNAL_HIGH       0x400C2000
+
+
 //Interrupt hardware source table
 //This table is decided by hardware, don't touch this.
 #define ETS_WIFI_MAC_INTR_SOURCE                0/**< interrupt of WiFi MAC, level*/

components/soc/esp32/soc_memory_layout.c

@@ -31,7 +31,7 @@ Each type contains an array of prioritised capabilities; types with later entrie
 ones can't fulfill the memory request.
 
 The prioritised capabilities work roughly like this:
-- For a normal malloc (MALLOC_CAP_8BIT), give away the DRAM-only memory first, then pass off any dual-use IRAM regions,
+- For a normal malloc (MALLOC_CAP_DEFAULT), give away the DRAM-only memory first, then pass off any dual-use IRAM regions,
   finally eat into the application memory.
 - For a malloc where 32-bit-aligned-only access is okay, first allocate IRAM, then DRAM, finally application IRAM.
 - Application mallocs (PIDx) will allocate IRAM first, if possible, then DRAM.
@@ -40,10 +40,10 @@ The prioritised capabilities work roughly like this:
 */
 const soc_memory_type_desc_t soc_memory_types[] = {
     //Type 0: Plain ole D-port RAM
-    { "DRAM", { MALLOC_CAP_DMA|MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL, MALLOC_CAP_32BIT, 0 }, false, false},
+    { "DRAM", { MALLOC_CAP_8BIT|MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL|MALLOC_CAP_DMA|MALLOC_CAP_32BIT, 0 }, false, false},
     //Type 1: Plain ole D-port RAM which has an alias on the I-port
     //(This DRAM is also the region used by ROM during startup)
-    { "D/IRAM", { 0, MALLOC_CAP_DMA|MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL, MALLOC_CAP_32BIT|MALLOC_CAP_EXEC }, true, true},
+    { "D/IRAM", { 0, MALLOC_CAP_DMA|MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL|MALLOC_CAP_DEFAULT, MALLOC_CAP_32BIT|MALLOC_CAP_EXEC }, true, true},
     //Type 2: IRAM
     { "IRAM", { MALLOC_CAP_EXEC|MALLOC_CAP_32BIT|MALLOC_CAP_INTERNAL, 0, 0 }, false, false},
     //Type 3-8: PID 2-7 IRAM
@@ -54,14 +54,14 @@ const soc_memory_type_desc_t soc_memory_types[] = {
     { "PID6IRAM", { MALLOC_CAP_PID6|MALLOC_CAP_INTERNAL, 0, MALLOC_CAP_EXEC|MALLOC_CAP_32BIT }, false, false},
     { "PID7IRAM", { MALLOC_CAP_PID7|MALLOC_CAP_INTERNAL, 0, MALLOC_CAP_EXEC|MALLOC_CAP_32BIT }, false, false},
     //Type 9-14: PID 2-7 DRAM
-    { "PID2DRAM", { MALLOC_CAP_PID2|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT }, false, false},
-    { "PID3DRAM", { MALLOC_CAP_PID3|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT }, false, false},
-    { "PID4DRAM", { MALLOC_CAP_PID4|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT }, false, false},
-    { "PID5DRAM", { MALLOC_CAP_PID5|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT }, false, false},
-    { "PID6DRAM", { MALLOC_CAP_PID6|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT }, false, false},
-    { "PID7DRAM", { MALLOC_CAP_PID7|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT }, false, false},
+    { "PID2DRAM", { MALLOC_CAP_PID2|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT|MALLOC_CAP_DEFAULT }, false, false},
+    { "PID3DRAM", { MALLOC_CAP_PID3|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT|MALLOC_CAP_DEFAULT }, false, false},
+    { "PID4DRAM", { MALLOC_CAP_PID4|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT|MALLOC_CAP_DEFAULT }, false, false},
+    { "PID5DRAM", { MALLOC_CAP_PID5|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT|MALLOC_CAP_DEFAULT }, false, false},
+    { "PID6DRAM", { MALLOC_CAP_PID6|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT|MALLOC_CAP_DEFAULT }, false, false},
+    { "PID7DRAM", { MALLOC_CAP_PID7|MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT, MALLOC_CAP_32BIT|MALLOC_CAP_DEFAULT }, false, false},
     //Type 15: SPI SRAM data
-    { "SPIRAM", { MALLOC_CAP_SPIRAM, 0, MALLOC_CAP_DMA|MALLOC_CAP_8BIT|MALLOC_CAP_32BIT}, false, false},
+    { "SPIRAM", { MALLOC_CAP_SPIRAM|MALLOC_CAP_DEFAULT, 0, MALLOC_CAP_8BIT|MALLOC_CAP_32BIT}, false, false},
 };
 
 const size_t soc_memory_type_count = sizeof(soc_memory_types)/sizeof(soc_memory_type_desc_t);

components/soc/include/soc/soc_memory_layout.h

@@ -17,6 +17,8 @@
 #include <stdbool.h>
 
 #include "soc/soc.h"
+#include "sdkconfig.h"
+#include "esp_attr.h"
 
 #define SOC_MEMORY_TYPE_NO_PRIOS 3
 
@@ -58,12 +60,12 @@ typedef struct
 extern const soc_reserved_region_t soc_reserved_regions[];
 extern const size_t soc_reserved_region_count;
 
-inline static bool esp_ptr_dma_capable(const void *p)
+inline static bool IRAM_ATTR esp_ptr_dma_capable(const void *p)
 {
     return (intptr_t)p >= SOC_DMA_LOW && (intptr_t)p < SOC_DMA_HIGH;
 }
 
-inline static bool esp_ptr_executable(const void *p)
+inline static bool IRAM_ATTR esp_ptr_executable(const void *p)
 {
     intptr_t ip = (intptr_t) p;
     return (ip >= SOC_IROM_LOW && ip < SOC_IROM_HIGH)
@@ -71,8 +73,19 @@ inline static bool esp_ptr_executable(const void *p)
         || (ip >= SOC_RTC_IRAM_LOW && ip < SOC_RTC_IRAM_HIGH);
 }
 
-inline bool esp_ptr_byte_accesible(const void *p)
+inline static bool IRAM_ATTR esp_ptr_byte_accessible(const void *p)
 {
-    //currently only support DRAM, add PSRAM region in the future
-    return (intptr_t)p >= SOC_BYTE_ACCESSIBLE_LOW && (intptr_t)p < SOC_BYTE_ACCESSIBLE_HIGH;
+    bool r;
+    r = ((intptr_t)p >= SOC_BYTE_ACCESSIBLE_LOW && (intptr_t)p < SOC_BYTE_ACCESSIBLE_HIGH);
+#if CONFIG_SPIRAM_SUPPORT
+    r |= ((intptr_t)p >= SOC_EXTRAM_DATA_LOW && (intptr_t)p < SOC_EXTRAM_DATA_HIGH);
+#endif
+    return r;
+}
+
+inline static bool IRAM_ATTR esp_ptr_internal(const void *p) {
+    bool r;
+    r = ((intptr_t)p >= SOC_MEM_INTERNAL_LOW && (intptr_t)p < SOC_MEM_INTERNAL_HIGH);
+    r |= ((intptr_t)p >= SOC_RTC_DATA_LOW && (intptr_t)p < SOC_RTC_DATA_HIGH);
+    return r;
 }

components/spi_flash/flash_mmap.c

@@ -135,7 +135,7 @@ esp_err_t IRAM_ATTR spi_flash_mmap_pages(int *pages, size_t page_count, spi_flas
             return ESP_ERR_INVALID_ARG;
         }
     }
-    mmap_entry_t* new_entry = (mmap_entry_t*) malloc(sizeof(mmap_entry_t));
+    mmap_entry_t* new_entry = (mmap_entry_t*) heap_caps_malloc(sizeof(mmap_entry_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
     if (new_entry == 0) {
         return ESP_ERR_NO_MEM;
     }

docs/api-guides/external-ram.rst

@@ -0,0 +1,104 @@
+Support for external RAM
+************************
+
+.. toctree::
+   :maxdepth: 1
+
+Introduction
+============
+
+The ESP32 has a few hundred KiB of internal RAM, residing on the same die as the rest of the ESP32. For some purposes, this is insufficient, 
+and therefore the ESP32 incorporates the ability to also use up to 4MiB of external SPI RAM memory as memory. The external memory is incorporated 
+in the memory map and is, within certain restrictions, usable in the same way internal data RAM is.
+
+Hardware
+========
+
+The ESP32 supports SPI (P)SRAM connected in parallel with the SPI flash chip. While the ESP32 is capable of supporting several types
+of RAM chips, the ESP32 SDK at the moment only supports the ESP-PSRAM32 chip.
+
+The ESP-PSRAM32 chip is an 1.8V device, and can only be used in parallel with an 1.8V flash part. Make sure to either set the MTDI 
+pin to a high signal level on bootup, or program the fuses in the ESP32 to always use a VDD_SIO level of 1.8V. Not doing this risks
+damaging the PSRAM and/or flash chip.
+
+To connect the ESP-PSRAM chip to the ESP32D0W*, connect the following signals:
+ * PSRAM /CE (pin 1) - ESP32 GPIO 16
+ * PSRAM SO (pin 2) - flash DO
+ * PSRAM SIO[2] (pin 3) - flash WP
+ * PSRAM SI (pin 5) - flash DI
+ * PSRAM SCLK (pin 6) - ESP32 GPIO 17
+ * PSRAM SIO[3] (pin 7) - flash HOLD
+ * PSRAM Vcc (pin 8) - ESP32 VCC_SDIO
+
+Connections for the ESP32D2W* chips are TBD.
+
+.. NOTE::
+   Espressif sells an ESP-WROVER module which contains an ESP32, 1.8V flash and the ESP-PSRAM32 integrated in a module, ready for inclusion
+   on an end product PCB.
+
+Software
+========
+
+ESP-IDF fully supports integrating external memory use into your applications. ESP-IDF can be configured to handle external RAM in several ways:
+ * Only initialize RAM. This allows the application to manually place data here by dereferencing pointers pointed at the external RAM memory
+   region (0x3F800000 and up).
+ * Initialize RAM and add it to the capability allocator. This allows a program to specifically allocate a chunk of external RAM using
+   ``heap_caps_malloc(size, MALLOC_CAP_SPIRAM)``. This memory can be used and subsequently freed using a normal ``free()`` call.
+ * Initialize RAM, add it to the capability allocator and add memory to the pool of RAM that can be returned by ``malloc()``. This allows
+   any application to use the external RAM without having to rewrite the code to use ``heap_caps_malloc``.
+
+All these options can be selected from the menuconfig menu.
+
+Restrictions
+------------
+
+The use of external RAM has a few restrictions:
+ * When disabling flash cache (for example, because the flash is being written to), the external RAM also becomes inaccessible; any reads from or
+   writes to it will lead to an illegal cache access exception. This is also the reason that ESP-IDF will never allocate a tasks stack in external
+   RAM.
+ * External RAM cannot be used as a place to store DMA transaction descriptors or as a buffer for a DMA transfer to read from or write into. Any
+   buffers that will be used in combination with DMA must be allocated using ``heap_caps_malloc(size, MALLOC_CAP_DMA)`` (and can be freed using a
+   standard ``free()`` call.)
+ * External RAM uses the same cache region as the external flash. This means that often accessed variables in external RAM can be read and 
+   modified almost as quickly as in internal ram. However, when accessing large chunks of data (>32K), the cache can be insufficient and speeds 
+   will fall back to the access speed of the external RAM. Moreover, accessing large chunks of data can 'push out' cached flash, possibly making 
+   execution of code afterwards slower.
+ * External RAM cannot be used as task stack memory; because of this, xTaskCreate and similar functions will always allocate internal memory
+   for stack and task TCBs and xTaskCreateStatic-type functions will check if the buffers passed are internal. However, for tasks not calling
+   on code in ROM in any way, directly or indirectly, the menuconfig option :ref:`CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY` will eliminate
+   the check in xTaskCreateStatic, allowing task stack in external RAM. Using this is not advised, however.
+
+
+Because there are a fair few situations that have a specific need for internal memory, but it is also possible to use malloc() to exhaust
+internal memory, there is a pool reserved specifically for requests that cannot be resolved from external memory; allocating task
+stack, DMA buffers and memory that stays accessible when cache is disabled is drawn from this pool. The size of this pool is configurable
+in menuconfig.
+
+
+Chip revisions
+==============
+
+There are some issues with certain revisions of the ESP32 that have repercussions for use with external RAM. These are documented in the ESP32 
+ECO_ document. In particular, ESP-IDF handles the bugs mentioned in the following ways:
+
+ESP32 rev v0
+------------
+ESP-IDF has no workaround for the bugs in this revision of silicon, and it cannot be used to map external PSRAM into the ESP32s main memory map.
+
+ESP32 rev v1
+------------
+The bugs in this silicon revision introduce a hazard when certain sequences of machine instructions operate on external memory locations (ESP32 ECO 3.2). 
+To work around this, the gcc compiler to compile ESP-IDF has been expanded with a flag: ``-mfix-esp32-psram-cache-issue``. With this flag passed to gcc 
+on the command line, the compiler works around these sequences and only outputs code that can safely be executed.
+
+In ESP-IDF, this flag is enabled when you select :ref:`CONFIG_SPIRAM_CACHE_WORKAROUND`. ESP-IDF also takes other measures to make
+sure no combination of PSRAM access plus the offending instruction sets are used: it links to a version of Newlib recompiled with the gcc flag, doesn't use
+some ROM functions and allocates static memory for the WiFi stack.
+
+.. _ECO: https://www.espressif.com/sites/default/files/documentation/eco_and_workarounds_for_bugs_in_esp32_en.pdf
+
+
+
+
+
+

docs/api-guides/index.rst

@@ -20,3 +20,4 @@ API Guides
    Console Component <console>
    ROM debug console <romconsole>
    WiFi Driver <wifi>
+   External SPI-connected RAM <external-ram>