Merge branch 'feature/likely_unlikely' into 'master'

Adding likely unlikely macros to hot code paths

Closes IDF-284

See merge request espressif/esp-idf!6368

components/bt/host/nimble/esp-hci/src/esp_nimble_hci.c

@@ -28,6 +28,7 @@
 #include "esp_nimble_hci.h"
 #include "esp_bt.h"
 #include "freertos/semphr.h"
+#include "esp_compiler.h"
 
 #define NIMBLE_VHCI_TIMEOUT_MS  2000
 

components/esp_common/include/esp_compiler.h

@@ -0,0 +1,33 @@
+// Copyright 2016-2019 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef __ESP_COMPILER_H
+#define __ESP_COMPILER_H
+
+/*
+ * The likely and unlikely macro pairs:
+ * These macros are useful to place when application  
+ * knows the majority ocurrence of a decision paths,
+ * placing one of these macros can hint the compiler
+ * to reorder instructions producing more optimized 
+ * code.
+ */ 
+#if (CONFIG_COMPILER_OPTIMIZATION_PERF)
+#define likely(x)      __builtin_expect(!!(x), 1) 
+#define unlikely(x)    __builtin_expect(!!(x), 0) 
+#else
+#define likely(x)      (x)
+#define unlikely(x)    (x) 
+#endif
+
+#endif

components/fatfs/diskio/diskio_rawflash.c

@@ -18,6 +18,7 @@
 #include "ff.h"
 #include "esp_log.h"
 #include "diskio_rawflash.h"
+#include "esp_compiler.h"
 
 static const char* TAG = "diskio_rawflash";
 
@@ -40,7 +41,7 @@ DRESULT ff_raw_read (BYTE pdrv, BYTE *buff, DWORD sector, UINT count)
     const esp_partition_t* part = ff_raw_handles[pdrv];
     assert(part);
     esp_err_t err = esp_partition_read(part, sector * SPI_FLASH_SEC_SIZE, buff, count * SPI_FLASH_SEC_SIZE);
-    if (err != ESP_OK) {
+    if (unlikely(err != ESP_OK)) {
         ESP_LOGE(TAG, "esp_partition_read failed (0x%x)", err);
         return RES_ERROR;
     }

components/fatfs/diskio/diskio_sdmmc.c

@@ -17,6 +17,7 @@
 #include "ff.h"
 #include "sdmmc_cmd.h"
 #include "esp_log.h"
+#include "esp_compiler.h"
 
 static sdmmc_card_t* s_cards[FF_VOLUMES] = { NULL };
 
@@ -37,7 +38,7 @@ DRESULT ff_sdmmc_read (BYTE pdrv, BYTE* buff, DWORD sector, UINT count)
     sdmmc_card_t* card = s_cards[pdrv];
     assert(card);
     esp_err_t err = sdmmc_read_sectors(card, buff, sector, count);
-    if (err != ESP_OK) {
+    if (unlikely(err != ESP_OK)) {
         ESP_LOGE(TAG, "sdmmc_read_blocks failed (%d)", err);
         return RES_ERROR;
     }
@@ -49,7 +50,7 @@ DRESULT ff_sdmmc_write (BYTE pdrv, const BYTE* buff, DWORD sector, UINT count)
     sdmmc_card_t* card = s_cards[pdrv];
     assert(card);
     esp_err_t err = sdmmc_write_sectors(card, buff, sector, count);
-    if (err != ESP_OK) {
+    if (unlikely(err != ESP_OK)) {
         ESP_LOGE(TAG, "sdmmc_write_blocks failed (%d)", err);
         return RES_ERROR;
     }

components/fatfs/diskio/diskio_wl.c

@@ -19,6 +19,7 @@
 #include "esp_log.h"
 #include "diskio_wl.h"
 #include "wear_levelling.h"
+#include "esp_compiler.h"
 
 static const char* TAG = "ff_diskio_spiflash";
 
@@ -43,7 +44,7 @@ DRESULT ff_wl_read (BYTE pdrv, BYTE *buff, DWORD sector, UINT count)
     wl_handle_t wl_handle = ff_wl_handles[pdrv];
     assert(wl_handle + 1);
     esp_err_t err = wl_read(wl_handle, sector * wl_sector_size(wl_handle), buff, count * wl_sector_size(wl_handle));
-    if (err != ESP_OK) {
+    if (unlikely(err != ESP_OK)) {
         ESP_LOGE(TAG, "wl_read failed (%d)", err);
         return RES_ERROR;
     }
@@ -56,12 +57,12 @@ DRESULT ff_wl_write (BYTE pdrv, const BYTE *buff, DWORD sector, UINT count)
     wl_handle_t wl_handle = ff_wl_handles[pdrv];
     assert(wl_handle + 1);
     esp_err_t err = wl_erase_range(wl_handle, sector * wl_sector_size(wl_handle), count * wl_sector_size(wl_handle));
-    if (err != ESP_OK) {
+    if (unlikely(err != ESP_OK)) {
         ESP_LOGE(TAG, "wl_erase_range failed (%d)", err);
         return RES_ERROR;
     }
     err = wl_write(wl_handle, sector * wl_sector_size(wl_handle), buff, count * wl_sector_size(wl_handle));
-    if (err != ESP_OK) {
+    if (unlikely(err != ESP_OK)) {
         ESP_LOGE(TAG, "wl_write failed (%d)", err);
         return RES_ERROR;
     }

components/freertos/include/freertos/FreeRTOS.h

@@ -94,6 +94,8 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+/* for likely and unlikely */
+#include "esp_compiler.h"
 
 /* Application specific configuration options. */
 #include "FreeRTOSConfig.h"

components/freertos/include/freertos/FreeRTOSConfig.h

@@ -126,12 +126,12 @@ int xt_clock_freq(void) __attribute__((deprecated));
 #if defined(CONFIG_FREERTOS_ASSERT_DISABLE)
 #define configASSERT(a) /* assertions disabled */
 #elif defined(CONFIG_FREERTOS_ASSERT_FAIL_PRINT_CONTINUE)
-#define configASSERT(a) if (!(a)) {                                     \
+#define configASSERT(a) if (unlikely(!(a))) {                                     \
         ets_printf("%s:%d (%s)- assert failed!\n", __FILE__, __LINE__,  \
                    __FUNCTION__);                                       \
     }
 #else /* CONFIG_FREERTOS_ASSERT_FAIL_ABORT */
-#define configASSERT(a) if (!(a)) {                                     \
+#define configASSERT(a) if (unlikely(!(a))) {                                     \
         ets_printf("%s:%d (%s)- assert failed!\n", __FILE__, __LINE__,  \
                    __FUNCTION__);                                       \
         abort();                                                        \

components/freertos/list.c

@@ -72,6 +72,7 @@
 #include "FreeRTOS.h"
 #include "list.h"
 
+
 /*-----------------------------------------------------------
  * PUBLIC LIST API documented in list.h
  *----------------------------------------------------------*/
@@ -215,7 +216,7 @@ List_t * const pxList = ( List_t * ) pxItemToRemove->pvContainer;
 	pxItemToRemove->pxPrevious->pxNext = pxItemToRemove->pxNext;
 
 	/* Make sure the index is left pointing to a valid item. */
-	if( pxList->pxIndex == pxItemToRemove )
+	if(pxList->pxIndex == pxItemToRemove)
 	{
 		pxList->pxIndex = pxItemToRemove->pxPrevious;
 	}

components/freertos/port.c

@@ -90,7 +90,6 @@
 // SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------------------
 */
-
 #include <stdlib.h>
 #include <string.h>
 #include <xtensa/config/core.h>
@@ -109,6 +108,7 @@
 #include "esp_intr_alloc.h"
 #include "esp_log.h"
 #include "sdkconfig.h"
+#include "esp_compiler.h"
 
 /* Defined in portasm.h */
 extern void _frxt_tick_timer_init(void);

components/freertos/queue.c

@@ -782,7 +782,7 @@ Queue_t * const pxQueue = ( Queue_t * ) xQueue;
 								mtCOVERAGE_TEST_MARKER();
 							}
 						}
-						else if( xYieldRequired != pdFALSE )
+						else if(xYieldRequired != pdFALSE)
 						{
 							/* This path is a special case that will only get
 							executed if the task was holding multiple mutexes
@@ -815,7 +815,7 @@ Queue_t * const pxQueue = ( Queue_t * ) xQueue;
 							mtCOVERAGE_TEST_MARKER();
 						}
 					}
-					else if( xYieldRequired != pdFALSE )
+					else if(xYieldRequired != pdFALSE)
 					{
 						/* This path is a special case that will only get
 						executed if the task was holding multiple mutexes and
@@ -868,7 +868,7 @@ Queue_t * const pxQueue = ( Queue_t * ) xQueue;
 		taskENTER_CRITICAL(&pxQueue->mux);
 
 		/* Update the timeout state to see if it has expired yet. */
-		if( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE )
+		if(xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE )
 		{
 			if( prvIsQueueFull( pxQueue ) != pdFALSE )
 			{
@@ -1573,7 +1573,7 @@ Queue_t * const pxQueue = ( Queue_t * ) xQueue;
 		taskENTER_CRITICAL(&pxQueue->mux);
 
 		/* Update the timeout state to see if it has expired yet. */
-		if( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE )
+		if(xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE)
 		{
 			if( prvIsQueueEmpty( pxQueue ) != pdFALSE )
 			{
@@ -1888,7 +1888,7 @@ BaseType_t xReturn = pdFALSE;
 		}
 		#endif /* configUSE_MUTEXES */
 	}
-	else if( xPosition == queueSEND_TO_BACK )
+	else if(xPosition == queueSEND_TO_BACK)
 	{
 		( void ) memcpy( ( void * ) pxQueue->pcWriteTo, pvItemToQueue, ( size_t ) pxQueue->uxItemSize ); /*lint !e961 !e418 MISRA exception as the casts are only redundant for some ports, plus previous logic ensures a null pointer can only be passed to memcpy() if the copy size is 0. */
 		pxQueue->pcWriteTo += pxQueue->uxItemSize;

components/freertos/tasks.c

@@ -77,6 +77,7 @@ all the API functions to use the MPU wrappers.  That should only be done when
 task.h is included from an application file. */
 #define MPU_WRAPPERS_INCLUDED_FROM_API_FILE
 #include "esp_newlib.h"
+#include "esp_compiler.h"
 
 /* FreeRTOS includes. */
 #include "FreeRTOS.h"
@@ -2466,7 +2467,7 @@ BaseType_t xSwitchRequired = pdFALSE;
 	/* Only allow core 0 increase the tick count in the case of xPortSysTickHandler processing. */
 	/* And allow core 0 and core 1 to unwind uxPendedTicks during xTaskResumeAll. */
 
-	if ( xPortInIsrContext() )
+	if (xPortInIsrContext())
 	{
 		#if ( configUSE_TICK_HOOK == 1 )
 		vApplicationTickHook();
@@ -4187,7 +4188,7 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po
 	{
 		BaseType_t oldInterruptLevel=0;
 		BaseType_t schedulerRunning = xSchedulerRunning;
-		if( schedulerRunning != pdFALSE )
+		if(schedulerRunning != pdFALSE)
 		{
 			//Interrupts may already be disabled (because we're doing this recursively) but we can't get the interrupt level after
 			//vPortCPUAquireMutex, because it also may mess with interrupts. Get it here first, then later figure out if we're nesting
@@ -4200,7 +4201,7 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po
 		vPortCPUAcquireMutexIntsDisabled( mux, portMUX_NO_TIMEOUT );
 #endif
 
-		if( schedulerRunning != pdFALSE )
+		if(schedulerRunning != pdFALSE)
 		{
 			TCB_t *tcb = pxCurrentTCB[xPortGetCoreID()];
 			BaseType_t newNesting = tcb->uxCriticalNesting + 1;
@@ -4259,11 +4260,11 @@ For ESP32 FreeRTOS, vTaskExitCritical implements both portEXIT_CRITICAL and port
 #else
 		vPortCPUReleaseMutexIntsDisabled( mux );
 #endif
-		if( xSchedulerRunning != pdFALSE )
+		if(xSchedulerRunning != pdFALSE)
 		{
 			TCB_t *tcb = pxCurrentTCB[xPortGetCoreID()];
 			BaseType_t nesting = tcb->uxCriticalNesting;
-			if( nesting	 > 0U )
+			if(nesting > 0U)
 			{
 				nesting--;
 				tcb->uxCriticalNesting = nesting;

components/lwip/port/esp32/freertos/sys_arch.c

@@ -40,6 +40,7 @@
 #include "arch/sys_arch.h"
 #include "lwip/stats.h"
 #include "esp_log.h"
+#include "esp_compiler.h"
 
 static const char* TAG = "lwip_arch";
 

components/lwip/port/esp32/netif/ethernetif.c

@@ -51,6 +51,7 @@
 #include "esp_eth.h"
 #include "esp_netif.h"
 #include "esp_netif_net_stack.h"
+#include "esp_compiler.h"
 
 /* Define those to better describe your network interface. */
 #define IFNAME0 'e'
@@ -133,7 +134,7 @@ static err_t ethernet_low_level_output(struct netif *netif, struct pbuf *p)
         pbuf_free(q);
     }
     /* Check error */
-    if (ret != ESP_OK) {
+    if (unlikely(ret != ESP_OK)) {
         return ERR_ABRT;
     } else {
         return ERR_OK;
@@ -156,7 +157,7 @@ void ethernetif_input(void *h, void *buffer, size_t len, void *eb)
     struct netif *netif = h;
     struct pbuf *p;
 
-    if (buffer == NULL || !netif_is_up(netif)) {
+    if (unlikely(buffer == NULL || !netif_is_up(netif))) {
         if (buffer) {
             ethernet_free_rx_buf_l2(netif, buffer);
         }
@@ -175,7 +176,7 @@ void ethernetif_input(void *h, void *buffer, size_t len, void *eb)
     p->l2_buf = buffer;
 #endif
     /* full packet send to tcpip_thread to process */
-    if (netif->input(p, netif) != ERR_OK) {
+    if (unlikely(netif->input(p, netif) != ERR_OK)) {
         LWIP_DEBUGF(NETIF_DEBUG, ("ethernetif_input: IP input error\n"));
         pbuf_free(p);
     }

components/lwip/port/esp32/netif/wlanif.c

@@ -52,6 +52,7 @@
 
 #include "esp_netif.h"
 #include "esp_netif_net_stack.h"
+#include "esp_compiler.h"
 
 /**
  * @brief Free resources allocated in L2 layer
@@ -160,7 +161,7 @@ wlanif_input(void *h, void *buffer, size_t len, void* eb)
   esp_netif_t *esp_netif = esp_netif_get_handle_from_netif_impl(netif);
   struct pbuf *p;
 
-  if(!buffer || !netif_is_up(netif)) {
+  if(unlikely(!buffer || !netif_is_up(netif))) {
     if (eb) {
       esp_netif_free_rx_buffer(esp_netif, eb);
     }
@@ -190,7 +191,7 @@ wlanif_input(void *h, void *buffer, size_t len, void* eb)
 #endif
 
   /* full packet send to tcpip_thread to process */
-  if (netif->input(p, netif) != ERR_OK) {
+  if (unlikely(netif->input(p, netif) != ERR_OK)) {
     LWIP_DEBUGF(NETIF_DEBUG, ("ethernetif_input: IP input error\n"));
     pbuf_free(p);
   }

components/newlib/platform_include/assert.h

@@ -19,10 +19,23 @@
 #pragma once
 #include <sdkconfig.h>
 #include <stdlib.h>
+#include "esp_compiler.h"
 
 #include_next <assert.h>
 
 #if defined(CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT) && !defined(NDEBUG)
-#undef assert
-#define assert(__e) ((__e) ? (void)0 : abort())
+    #undef assert
+    #define assert(__e) (likely(__e)) ? (void)0 : abort()
+#else 
+    /* moved part of toolchain provided assert to there then
+     * we can tweak the original assert macro to perform likely
+     * before deliver it to original toolchain implementation 
+     */
+    #undef assert
+    #ifdef NDEBUG 
+    # define assert(__e) ((void)0)
+    #else
+    # define assert(__e) (likely(__e) ? (void)0 : __assert_func (__FILE__, __LINE__, \
+                                __ASSERT_FUNC, #__e))
+    #endif                            
 #endif

components/sdmmc/sdmmc_io.c

@@ -17,6 +17,7 @@
 
 #include "sdmmc_common.h"
 #include "esp_attr.h"
+#include "esp_compiler.h"
 
 
 #define CIS_TUPLE(NAME)  (cis_tuple_t) {.code=CISTPL_CODE_##NAME, .name=#NAME, .func=&cis_tuple_func_default, }
@@ -235,7 +236,7 @@ esp_err_t sdmmc_io_read_byte(sdmmc_card_t* card, uint32_t function,
         uint32_t addr, uint8_t *out_byte)
 {
     esp_err_t ret = sdmmc_io_rw_direct(card, function, addr, SD_ARG_CMD52_READ, out_byte);
-    if (ret != ESP_OK) {
+    if (unlikely(ret != ESP_OK)) {
         ESP_LOGE(TAG, "%s: sdmmc_io_rw_direct (read 0x%x) returned 0x%x", __func__, addr, ret);
     }
     return ret;
@@ -247,7 +248,7 @@ esp_err_t sdmmc_io_write_byte(sdmmc_card_t* card, uint32_t function,
     uint8_t tmp_byte = in_byte;
     esp_err_t ret = sdmmc_io_rw_direct(card, function, addr,
             SD_ARG_CMD52_WRITE | SD_ARG_CMD52_EXCHANGE, &tmp_byte);
-    if (ret != ESP_OK) {
+    if (unlikely(ret != ESP_OK)) {
         ESP_LOGE(TAG, "%s: sdmmc_io_rw_direct (write 0x%x) returned 0x%x", __func__, addr, ret);
         return ret;
     }
@@ -323,7 +324,7 @@ esp_err_t sdmmc_io_read_bytes(sdmmc_card_t* card, uint32_t function,
         esp_err_t err = sdmmc_io_rw_extended(card, function, addr,
                 SD_ARG_CMD53_READ | SD_ARG_CMD53_INCREMENT,
                 pc_dst, will_transfer);
-        if (err != ESP_OK) {
+        if (unlikely(err != ESP_OK)) {
             return err;
         }
         pc_dst += will_transfer;
@@ -346,7 +347,7 @@ esp_err_t sdmmc_io_write_bytes(sdmmc_card_t* card, uint32_t function,
         esp_err_t err = sdmmc_io_rw_extended(card, function, addr,
                 SD_ARG_CMD53_WRITE | SD_ARG_CMD53_INCREMENT,
                 (void*) pc_src, will_transfer);
-        if (err != ESP_OK) {
+        if (unlikely(err != ESP_OK)) {
             return err;
         }
         pc_src += will_transfer;
@@ -359,7 +360,7 @@ esp_err_t sdmmc_io_write_bytes(sdmmc_card_t* card, uint32_t function,
 esp_err_t sdmmc_io_read_blocks(sdmmc_card_t* card, uint32_t function,
         uint32_t addr, void* dst, size_t size)
 {
-    if (size % 4 != 0) {
+    if (unlikely(size % 4 != 0)) {
         return ESP_ERR_INVALID_SIZE;
     }
     return sdmmc_io_rw_extended(card, function, addr,
@@ -370,7 +371,7 @@ esp_err_t sdmmc_io_read_blocks(sdmmc_card_t* card, uint32_t function,
 esp_err_t sdmmc_io_write_blocks(sdmmc_card_t* card, uint32_t function,
         uint32_t addr, const void* src, size_t size)
 {
-    if (size % 4 != 0) {
+    if (unlikely(size % 4 != 0)) {
         return ESP_ERR_INVALID_SIZE;
     }
     return sdmmc_io_rw_extended(card, function, addr,

components/spiffs/spiffs_api.c

@@ -35,7 +35,7 @@ s32_t spiffs_api_read(spiffs *fs, uint32_t addr, uint32_t size, uint8_t *dst)
 {
     esp_err_t err = esp_partition_read(((esp_spiffs_t *)(fs->user_data))->partition, 
                                         addr, dst, size);
-    if (err) {
+    if (unlikely(err)) {
         ESP_LOGE(TAG, "failed to read addr %08x, size %08x, err %d", addr, size, err);
         return -1;
     }
@@ -46,7 +46,7 @@ s32_t spiffs_api_write(spiffs *fs, uint32_t addr, uint32_t size, uint8_t *src)
 {
     esp_err_t err = esp_partition_write(((esp_spiffs_t *)(fs->user_data))->partition, 
                                         addr, src, size);
-    if (err) {
+    if (unlikely(err)) {
         ESP_LOGE(TAG, "failed to write addr %08x, size %08x, err %d", addr, size, err);
         return -1;
     }

components/spiffs/spiffs_api.h

@@ -21,6 +21,7 @@
 #include "freertos/semphr.h"
 #include "spiffs.h"
 #include "esp_vfs.h"
+#include "esp_compiler.h"
 
 #ifdef __cplusplus
 extern "C" {