cmake: Add embedding files in components support

Add subscribe_publish AWS example and fixes to allow it to build.

components/esp32/CMakeLists.txt

@@ -10,6 +10,8 @@ if(BOOTLOADER_BUILD)
 else()
   # Regular app build
 
+  set(COMPONENT_SRCDIRS ". hwcrypto")
+
   register_component()
 
   target_link_libraries(esp32 "-L ${CMAKE_CURRENT_SOURCE_DIR}/lib")

components/mbedtls/CMakeLists.txt

@@ -4,5 +4,5 @@ set(COMPONENT_SRCDIRS library port)
 register_component()
 
 target_compile_definitions(mbedtls PUBLIC
-	MBEDTLS_CONFIG_FILE="mbedtls/esp_config.h"
+	-DMBEDTLS_CONFIG_FILE="mbedtls/esp_config.h"
 )

examples/protocols/aws_iot/subscribe_publish/CMakeLists.txt

@@ -0,0 +1,18 @@
+# The following four lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+set(CMAKE_TOOLCHAIN_FILE $ENV{IDF_PATH}/toolchain.cmake)
+project(idf_project ASM C CXX)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+add_executable(subscribe_publish.elf
+  main/subscribe_publish_sample.c)
+
+target_add_binary_data(subscribe_publish.elf main/certs/aws-root-ca.pem TEXT)
+target_add_binary_data(subscribe_publish.elf main/certs/certificate.pem.crt TEXT)
+target_add_binary_data(subscribe_publish.elf main/certs/private.pem.key TEXT)
+
+# TODO: handle IDF_CI_BUILD dummy cert files here
+
+target_link_libraries(subscribe_publish.elf ${COMPONENT_LIBRARIES})

tools/cmake/components.cmake

@@ -76,14 +76,26 @@ function(register_component)
     endforeach()
   endif()
 
+  # binaries to embed directly in library
+  spaces2list(COMPONENT_EMBED_FILES)
+  spaces2list(COMPONENT_EMBED_TXTFILES)
+  foreach(embed_data ${COMPONENT_EMBED_FILES} ${COMPONENT_EMBED_TXTFILES})
+    if (embed_data IN_LIST COMPONENT_EMBED_TXTFILES)
+      set(embed_type "TEXT")
+    else()
+      set(embed_type "BINARY")
+    endif()
+    target_add_binary_data("${component}" "${embed_data}" "${embed_type}")
+  endforeach()
+
   # add public includes from other components when building this component
-  if(COMPONENT_SRCS)
-    add_library(${component} STATIC ${COMPONENT_SRCS})
+  if(COMPONENT_SRCS OR embed_binaries)
+    add_library(${component} STATIC ${COMPONENT_SRCS} ${embed_binaries})
     set(include_type PUBLIC)
   else()
     add_library(${component} INTERFACE) # header-only component
     set(include_type INTERFACE)
-  endif(COMPONENT_SRCS)
+  endif(COMPONENT_SRCS OR embed_binaries)
 
   foreach(include_dir ${COMPONENT_ADD_INCLUDEDIRS})
     get_filename_component(include_dir ${include_dir} ABSOLUTE BASE_DIR ${component_dir})
@@ -117,9 +129,13 @@ function(components_finish_registration)
       if (${a_type} STREQUAL STATIC_LIBRARY)
         foreach(b ${COMPONENTS})
           if (TARGET ${b} AND NOT ${a} STREQUAL ${b})
+            # Add all public compile options from b in a
             target_include_directories(${a} PRIVATE
-              $<TARGET_PROPERTY:${b},INTERFACE_INCLUDE_DIRECTORIES>
-              )
+              $<TARGET_PROPERTY:${b},INTERFACE_INCLUDE_DIRECTORIES>)
+            target_compile_definitions(${a} PRIVATE
+              $<TARGET_PROPERTY:${b},INTERFACE_COMPILE_DEFINITIONS>)
+            target_compile_options(${a} PRIVATE
+              $<TARGET_PROPERTY:${b},INTERFACE_COMPILE_OPTIONS>)
           endif()
         endforeach(b)
       endif(${a_type} STREQUAL STATIC_LIBRARY)
@@ -130,4 +146,15 @@ function(components_finish_registration)
 
   # set COMPONENT_LIBRARIES in top-level scope
   set(COMPONENT_LIBRARIES "${COMPONENT_LIBRARIES}" PARENT_SCOPE)
+
+  # Embedded binary & text files
+  spaces2list(COMPONENT_EMBED_FILES)
+  foreach(embed_src ${COMPONENT_EMBED_FILES})
+    target_add_binary_data(${component} "${embed_src}" BINARY)
+  endforeach()
+  spaces2list(COMPONENT_EMBED_TXTFILES)
+  foreach(embed_src ${COMPONENT_EMBED_TXTFILES})
+    target_add_binary_data(${component} "${embed_src}" TEXT)
+  endforeach()
+
 endfunction(components_finish_registration)

tools/cmake/idf_functions.cmake

@@ -69,7 +69,7 @@ function(idf_set_global_compiler_options)
   endif()
 
   # Always generate debug symbols (even in Release mode, these don't
-  # go itno ther final binary
+  # go into the final binary so have no impact on size)
   add_compile_options(-ggdb)
 
   add_compile_options("-I${CMAKE_BINARY_DIR}") # for sdkconfig.h
@@ -107,6 +107,4 @@ function(idf_verify_environment)
   gcc_version_check("5.2.0")
   crosstool_version_check("1.22.0-80-g6c4433a")
 
-  
-
-endfunction(idf_verify_environment)
+endfunction()

tools/cmake/kconfig.cmake

@@ -31,7 +31,7 @@ function(kconfig_process_config)
 
   # Find Kconfig and Kconfig.projbuild for each component as applicable
   # if any of these change, cmake should rerun
-  foreach(dir ${COMPONENT_PATHS})
+  foreach(dir ${COMPONENT_PATHS} "${CMAKE_SOURCE_DIR}/main")
     file(GLOB kconfig "${dir}/Kconfig")
     if(kconfig)
       set(kconfigs "${kconfigs} ${kconfig}")

tools/cmake/scripts/data_file_to_c.cmake

@@ -0,0 +1,56 @@
+#
+# Convert a file (text or binary) into a C source file suitable
+# for gcc. Designed to replicate 'objcopy' with more predictable
+# naming, and supports appending a null byte for embedding text as
+# a string.
+#
+# Designed to be run as a script with "cmake -P"
+#
+# Set variables DATA_FILE, SOURCE_FILE, FILE_TYPE when running this.
+#
+# If FILE_TYPE is set to STRING, a null byte is appended to DATA_FILE's contents
+# before it is embedded.
+#
+# If FILE_TYPE is unset (or any other value), DATA_FILE is embedded
+# verbatim.
+#
+#
+if(NOT DATA_FILE)
+  message(FATAL_ERROR "DATA_FILE for converting must be specified")
+endif()
+
+if(NOT SOURCE_FILE)
+  message(FATAL_ERROR "SOURCE_FILE destination must be specified")
+endif()
+
+file(READ "${DATA_FILE}" data HEX)
+
+if(FILE_TYPE STREQUAL "TEXT")
+  set(data "${data}00")  # null-byte terimnation
+endif()
+
+## Convert string of raw hex bytes to lines of hex bytes in C array format
+string(REGEX REPLACE "................................" "\\0\n    " data "${data}")  # 16 bytes per line
+string(REGEX REPLACE "[0-9a-f][0-9a-f]" "0x\\0, " data "${data}")                    # hex formatted C bytes
+string(REGEX REPLACE ", $" "" data "${data}")                                        # trim the last comma (cosmetic)
+
+## Come up with C-friendly name for file
+get_filename_component(source_filename "${DATA_FILE}" NAME)
+string(MAKE_C_IDENTIFIER "${source_filename}" varname)
+
+file(WRITE "${SOURCE_FILE}"  "/*\n")
+file(APPEND "${SOURCE_FILE}" " * Data converted from ${DATA_FILE}\n")
+if(FILE_TYPE STREQUAL "TEXT")
+  file(APPEND "${SOURCE_FILE}" " * (null byte appended)\n")
+endif()
+file(APPEND "${SOURCE_FILE}" " */\n")
+file(APPEND "${SOURCE_FILE}" "#include <stdlib.h>\n")
+file(APPEND "${SOURCE_FILE}" "#include <stddef.h>\n\n")
+
+file(APPEND "${SOURCE_FILE}" "const char ${varname}[] = {\n    ${data}\n    };\n")
+file(APPEND "${SOURCE_FILE}" "const size_t ${varname}_length = sizeof(${varname});\n\n")
+
+# Backwards compatibility, matches objcopy binary symbol names
+file(APPEND "${SOURCE_FILE}" "/* Backwards compatible names, match objcopy -I binary naming */\n")
+file(APPEND "${SOURCE_FILE}" "const char *_binary_${varname}_start = ${varname};\n")
+file(APPEND "${SOURCE_FILE}" "const char *_binary_${varname}_end = ${varname} + sizeof(${varname});\n")

tools/cmake/utilities.cmake

@@ -73,6 +73,7 @@ function(move_if_different source destination)
 
 endfunction()
 
+
 # add_compile_options variant for C++ code only
 #
 # This adds global options, set target properties for
@@ -94,3 +95,29 @@ function(add_c_compile_options)
     add_compile_options($<$<COMPILE_LANGUAGE:C>:${option}>)
   endforeach()
 endfunction()
+
+
+# target_add_binary_data adds binary data into the built target,
+# by converting it to a generated source file which is then compiled
+# to a binary object as part of the build
+function(target_add_binary_data target embed_file embed_type)
+
+  get_filename_component(embed_file "${embed_file}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+
+  get_filename_component(name "${embed_file}" NAME)
+  set(embed_srcfile "${CMAKE_BINARY_DIR}/${name}.c")
+
+  add_custom_command(OUTPUT "${embed_srcfile}"
+    COMMAND "${CMAKE_COMMAND}"
+    -D "DATA_FILE=${embed_file}"
+    -D "SOURCE_FILE=${embed_srcfile}"
+    -D "FILE_TYPE=${embed_type}"
+    -P "${IDF_PATH}/tools/cmake/scripts/data_file_to_c.cmake"
+    MAIN_DEPENDENCY "${embed_file}"
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
+
+  set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${embed_srcfile}")
+
+  target_sources("${target}" PRIVATE "${embed_srcfile}")
+endfunction()
+