Merge remote-tracking branch 'origin/master' into feature/github-7022

.flake8

@@ -164,3 +164,7 @@ exclude =
         components/wifi_provisioning/python/wifi_constants_pb2.py,
         components/esp_local_ctrl/python/esp_local_ctrl_pb2.py,
         examples/provisioning/legacy/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
+
+per-file-ignores =
+    # Sphinx conf.py files use star imports to setup config variables
+        docs/conf_common.py: F405

.github/workflows/python_lint.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8]
 
     steps:
       - name: Checkout

.gitignore

@@ -88,3 +88,6 @@ build
 
 # lock files for examples and components
 dependencies.lock
+
+# managed_components for examples
+managed_components

.gitlab-ci.yml

@@ -3,8 +3,8 @@ stages:
   - build
   - assign_test
   - build_doc
-  - host_test
   - target_test
+  - host_test
   - test_deploy
   - deploy
   - post_deploy
@@ -48,6 +48,11 @@ variables:
 
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v4"
+  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env:v4.4-1"
+  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test:v4.4-1-1"
+  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-2"
+  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
 
   # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
@@ -57,15 +62,12 @@ variables:
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v3.1"
 
-  # Versioned esp-idf-doc env image to use for all document building jobs
-  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v10"
-
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
   export IDF_MIRROR_PREFIX_MAP=
   fi
   if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
-  tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
+  tools/idf_tools.py --non-interactive install ${SETUP_TOOLS_LIST:-} && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
     if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
     echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
     wget $OOCD_DISTRO_URL
@@ -100,6 +102,7 @@ before_script:
 .before_script_macos:
   before_script:
     - source tools/ci/utils.sh
+    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
     - $IDF_PATH/tools/idf_tools.py install-python-env
     # On macOS, these tools need to be installed
     - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja

.gitlab/CODEOWNERS

@@ -47,8 +47,8 @@
 * @esp-idf-codeowners/other
 
 /.*                                   @esp-idf-codeowners/tools
-/.gitlab/ci/                          @esp-idf-codeowners/ci
 /.gitlab-ci.yml                       @esp-idf-codeowners/ci
+/.gitlab/ci/                          @esp-idf-codeowners/ci
 /.pre-commit-config.yaml              @esp-idf-codeowners/ci
 /.readthedocs.yml                     @esp-idf-codeowners/docs
 /CMakeLists.txt                       @esp-idf-codeowners/build-config
@@ -57,6 +57,7 @@
 /export.*                             @esp-idf-codeowners/tools
 /install.*                            @esp-idf-codeowners/tools
 /sdkconfig.rename                     @esp-idf-codeowners/build-config
+/sonar-project.properties             @esp-idf-codeowners/ci
 
 # sort-order-reset
 
@@ -66,6 +67,7 @@
 /components/bootloader*/              @esp-idf-codeowners/system @esp-idf-codeowners/security
 /components/bt/                       @esp-idf-codeowners/bluetooth
 /components/cbor/                     @esp-idf-codeowners/app-utilities
+/components/cmock/                    @esp-idf-codeowners/system
 /components/coap/                     @esp-idf-codeowners/app-utilities
 /components/console/                  @esp-idf-codeowners/system @esp-idf-codeowners/app-utilities
 /components/cxx/                      @esp-idf-codeowners/system
@@ -84,11 +86,13 @@
 /components/esp_https_server/         @esp-idf-codeowners/app-utilities
 /components/esp_hw_support/           @esp-idf-codeowners/system
 /components/esp_ipc/                  @esp-idf-codeowners/system
+/components/esp_lcd/                  @esp-idf-codeowners/peripherals
 /components/esp_local_ctrl/           @esp-idf-codeowners/app-utilities
 /components/esp_netif/                @esp-idf-codeowners/network
-/components/esp_pm/                   @esp-idf-codeowners/power-management
+/components/esp_phy/                  @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
+/components/esp_pm/                   @esp-idf-codeowners/power-management @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi
 /components/esp_ringbuf/              @esp-idf-codeowners/system
-/components/esp_rom/                  @esp-idf-codeowners/system
+/components/esp_rom/                  @esp-idf-codeowners/system @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi
 /components/esp_serial_slave_link/    @esp-idf-codeowners/peripherals
 /components/esp_system/               @esp-idf-codeowners/system
 /components/esp_timer/                @esp-idf-codeowners/system
@@ -104,6 +108,7 @@
 /components/hal/                      @esp-idf-codeowners/peripherals
 /components/heap/                     @esp-idf-codeowners/system
 /components/idf_test/                 @esp-idf-codeowners/ci
+/components/ieee802154/               @esp-idf-codeowners/ieee802154
 /components/jsmn/                     @esp-idf-codeowners/app-utilities
 /components/json/                     @esp-idf-codeowners/app-utilities
 /components/libsodium/                @esp-idf-codeowners/security
@@ -123,23 +128,31 @@
 /components/protocomm/                @esp-idf-codeowners/app-utilities
 /components/pthread/                  @esp-idf-codeowners/system
 /components/sdmmc/                    @esp-idf-codeowners/storage
-/components/soc/                      @esp-idf-codeowners/peripherals
+/components/soc/                      @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
 /components/spi_flash/                @esp-idf-codeowners/peripherals
 /components/spiffs/                   @esp-idf-codeowners/storage
 /components/tcp_transport/            @esp-idf-codeowners/network
 /components/tcpip_adapter/            @esp-idf-codeowners/network
-/components/tinyusb/                  @esp-idf-codeowners/peripherals
+/components/tinyusb/                  @esp-idf-codeowners/peripherals/usb
+/components/touch_element/            @esp-idf-codeowners/peripherals
 /components/ulp/                      @esp-idf-codeowners/system
 /components/unity/                    @esp-idf-codeowners/ci
+/components/usb/                      @esp-idf-codeowners/peripherals/usb
 /components/vfs/                      @esp-idf-codeowners/storage
 /components/wear_levelling/           @esp-idf-codeowners/storage
 /components/wifi_provisioning/        @esp-idf-codeowners/app-utilities
 /components/wpa_supplicant/           @esp-idf-codeowners/wifi
 /components/xtensa/                   @esp-idf-codeowners/system
 
-/docs/    @esp-idf-codeowners/docs
-
-/examples/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
+/docs/                                  @esp-idf-codeowners/docs
+/docs/**/api-reference/bluetooth/       @esp-idf-codeowners/bluetooth
+/docs/**/api-reference/network/         @esp-idf-codeowners/network
+/docs/**/api-reference/peripherals/     @esp-idf-codeowners/peripherals
+/docs/**/api-reference/protocols/       @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
+/docs/**/api-reference/provisioning/    @esp-idf-codeowners/app-utilities
+/docs/**/api-reference/storage/         @esp-idf-codeowners/storage
+/docs/**/api-reference/system/          @esp-idf-codeowners/system
+/docs/**/security/                      @esp-idf-codeowners/security
 
 /examples/bluetooth/          @esp-idf-codeowners/bluetooth
 /examples/build_system/       @esp-idf-codeowners/build-config
@@ -148,6 +161,7 @@
 /examples/ethernet/           @esp-idf-codeowners/network
 /examples/get-started/        @esp-idf-codeowners/system
 /examples/mesh/               @esp-idf-codeowners/wifi
+/examples/network/            @esp-idf-codeowners/network @esp-idf-codeowners/wifi
 /examples/openthread/         @esp-idf-codeowners/ieee802154
 /examples/peripherals/        @esp-idf-codeowners/peripherals
 /examples/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
@@ -157,6 +171,8 @@
 /examples/system/             @esp-idf-codeowners/system
 /examples/wifi/               @esp-idf-codeowners/wifi
 
+/examples/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
+
 /make/    @esp-idf-codeowners/build-config
 
 /tools/                       @esp-idf-codeowners/tools
@@ -170,5 +186,16 @@
 /tools/kconfig*/              @esp-idf-codeowners/build-config
 /tools/ldgen/                 @esp-idf-codeowners/build-config
 /tools/mass_mfg/              @esp-idf-codeowners/app-utilities
+/tools/mocks/                 @esp-idf-codeowners/system
+
+## Note: owners here should be the same as the owners for the same example subdir, above
+/tools/test_apps/build_system/       @esp-idf-codeowners/build-config
+/tools/test_apps/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
+/tools/test_apps/security/           @esp-idf-codeowners/security
+/tools/test_apps/system/             @esp-idf-codeowners/system
+
+/tools/test_apps/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
+
+/tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
 
 requirements.txt              @esp-idf-codeowners/tools

.gitlab/ci/assign-test.yml

@@ -10,6 +10,7 @@ assign_test:
     - build_ssc_esp32
     - build_esp_idf_tests_cmake_esp32
     - build_esp_idf_tests_cmake_esp32s2
+    - build_esp_idf_tests_cmake_esp32s3
     - build_esp_idf_tests_cmake_esp32c3
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
@@ -58,6 +59,7 @@ update_test_cases:
   needs:
     - build_esp_idf_tests_cmake_esp32
     - build_esp_idf_tests_cmake_esp32s2
+    - build_esp_idf_tests_cmake_esp32s3
     - build_esp_idf_tests_cmake_esp32c3
   artifacts:
     when: always

.gitlab/ci/build.yml

@@ -1,6 +1,6 @@
 .build_template:
   stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - build
   variables:
@@ -105,7 +105,7 @@ build_ssc_esp32c3:
     OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
     BUILD_SYSTEM: "cmake"
     TEST_TYPE: "unit_test"
-    PYTHON_VER: 3
+    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
   script:
     - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
@@ -156,7 +156,7 @@ build_esp_idf_tests_cmake_esp32c3:
     TEST_TYPE: example_test
     LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
     BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
-    PYTHON_VER: 3
+    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"
@@ -167,6 +167,7 @@ build_examples_make:
   extends:
     - .build_examples_template
     - .rules:build:example_test-esp32
+  stage: host_test
   # This is a workaround for a rarely encountered issue with building examples in CI.
   # Probably related to building of Kconfig in 'make clean' stage
   retry: 1
@@ -220,6 +221,14 @@ build_examples_cmake_esp32s2:
   variables:
     IDF_TARGET: esp32s2
 
+build_examples_cmake_esp32s3:
+  extends:
+    - .build_examples_cmake_template
+    - .rules:build:example_test-esp32s3
+  parallel: 8
+  variables:
+    IDF_TARGET: esp32s3
+
 build_examples_cmake_esp32c3:
   extends:
     - .build_examples_cmake_template
@@ -289,6 +298,13 @@ build_component_ut_esp32s2:
   variables:
     IDF_TARGET: esp32s2
 
+build_component_ut_esp32s3:
+  extends:
+    - .build_component_ut_template
+    - .rules:build:component_ut-esp32s3
+  variables:
+    IDF_TARGET: esp32s3
+
 build_component_ut_esp32c3:
   extends:
     - .build_component_ut_template
@@ -297,6 +313,7 @@ build_component_ut_esp32c3:
     IDF_TARGET: esp32c3
 
 .test_build_system_template:
+  stage: host_test
   extends:
     - .build_template
     - .rules:build
@@ -334,7 +351,8 @@ build_docker:
   extends:
     - .before_script_minimal
     - .rules:build:docker
-  stage: build
+  stage: host_test
+  needs: []
   image: espressif/docker-builder:1
   tags:
     - build_docker_amd64_brno
@@ -356,7 +374,8 @@ build_docker:
   extends:
     - .before_script_minimal
     - .rules:build:windows
-  stage: build
+  stage: host_test
+  needs: []
   image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
   tags:
     - build
@@ -376,32 +395,12 @@ build_idf_exe:
   variables:
     TEST_DIR: tools/windows/idf_exe
 
-build_cmdlinerunner:
-  extends: .test-on-windows
-  artifacts:
-    paths:
-      - tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll
-    expire_in: 4 days
-  variables:
-    TEST_DIR: tools/windows/tool_setup/cmdlinerunner
-
-build_installer:
-  extends: .test-on-windows
-  # using a different stage here to be able to use artifacts from build_cmdlinerunner job
-  stage: host_test
-  image: $CI_DOCKER_REGISTRY/wine-innosetup:2
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - build_cmdlinerunner
-  script:
-    - cd tools/windows/tool_setup/
-    - ./build_installer.sh
-
 # This job builds template app with permutations of targets and optimization levels
 build_template_app:
   extends:
     - .build_template_app_template
     - .rules:build
+  stage: host_test
   needs:
     - job: fast_template_app
       artifacts: false

.gitlab/ci/dependencies/dependencies.yml

@@ -49,22 +49,6 @@
     - build_system
     - macos
 
-"build:docs":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  patterns:
-    - docs
-  deploy:
-    - production
-
-"build:docs:label-only":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  deploy:
-    - preview
-
 # ---------------
 # Build Test Jobs
 # ---------------
@@ -88,15 +72,6 @@ build:integration_test:
 # -------------
 # Special Cases
 # -------------
-"build:example_test-esp32":
-  labels:
-    - build
-    - weekend_test  # only have esp32 jobs
-    - iperf_stress_test  # only have esp32 jobs
-  patterns:
-    - build_components
-    - build_system
-
 "build:example_test-esp32c3":  # esp32c3 test is only run by label, but build jobs should always be triggered
   labels:
     - build
@@ -183,13 +158,27 @@ build:integration_test:
 #################################
 # Triggered Only By Labels Jobs #
 #################################
-"labels:{0}":
-  matrix:
-    - [weekend_test, iperf_stress_test, nvs_coverage]
+"labels:iperf_stress_test": # example_test
   labels:
-    - "{0}"
+    - iperf_stress_test
+  included_in:
+    - build:example_test-esp32
+    - build:target_test
+    - test:any_test
+
+"labels:weekend_test": # custom test
+  labels:
+    - weekend_test
+  included_in:
+    - build:custom_test-esp32
+    - build:target_test
+    - test:any_test
+
+"labels:nvs_coverage": # host_test
+  labels:
+    - nvs_coverage
 
-"labels:fuzzer_test-weekend_test":
+"labels:fuzzer_test-weekend_test": # host test
   labels:
     - fuzzer_test
     - weekend_test

.gitlab/ci/deploy.yml

@@ -1,16 +1,16 @@
 .deploy_job_template:
   extends: .before_script_no_sync_submodule
   stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - deploy
-  dependencies: []
 
 push_to_github:
   extends:
     - .deploy_job_template
     - .before_script_minimal
     - .rules:protected-no_label
+  dependencies: []
   script:
     - add_github_ssh_keys
     - git remote remove github &>/dev/null || true
@@ -31,7 +31,6 @@ deploy_test_result:
       - ${CI_PROJECT_DIR}/test-management/*.log
       # save all test logs as artifacts, make it easier to track errors
       - ${CI_PROJECT_DIR}/TEST_LOGS
-      - $CI_PROJECT_DIR/$CI_COMMIT_SHA
     expire_in: 1 mos
   variables:
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
@@ -51,7 +50,7 @@ deploy_test_result:
     - cd test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
     # update test results
-    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS ${CI_PROJECT_DIR}/${CI_COMMIT_SHA} --pipeline_url ${CI_PIPELINE_URL}
+    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL}
 
 check_submodule_sync:
   extends:
@@ -64,6 +63,7 @@ check_submodule_sync:
     GIT_STRATEGY: clone
     SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
+  dependencies: []
   script:
     - git submodule deinit --force .
     # setting the default remote URL to the public one, to resolve relative location URLs

.gitlab/ci/docs.yml

@@ -1,8 +1,44 @@
+.patterns-docs: &patterns-docs
+  - ".gitlab/ci/docs.yml"
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig*"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "tools/kconfig_new/**/*"
+  - "CONTRIBUTING.rst"
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs: &if-label-docs
+  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs_full: &if-label-docs_full
+  if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.doc-rules:build:docs:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-label-docs_full
+    - <<: *if-dev-push
+      changes: *patterns-docs
+
 # stage: pre_check
 check_readme_links:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
   tags: ["build", "amd64", "internet"]
   allow_failure: true
   script:
@@ -11,41 +47,44 @@ check_readme_links:
 check_docs_lang_sync:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
     - ./check_lang_folder_sync.sh
 
 .build_docs_template:
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.13
   tags:
     - build_docs
   dependencies: []
   script:
     - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build
+    - pip install -r requirements.txt
+    - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
   parallel:
     matrix:
       - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2", "esp32c3"]
+        DOCTGT: ["esp32", "esp32s2", "esp32s3", "esp32c3"]
 
 check_docs_gh_links:
+  image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.13
   extends:
     - .pre_check_job_template
-    - .build_docs_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
+    - pip install -r requirements.txt
+    - build-docs gh-linkcheck
 
 # stage: build_doc
 # Add this stage to let the build_docs job run in parallel with build
 .build_docs_build_stage_template:
   extends:
     - .build_docs_template
-    - .rules:build:docs
   stage: build_doc
   needs:
     - job: check_docs_lang_sync
@@ -53,9 +92,33 @@ check_docs_gh_links:
     - job: check_docs_gh_links
       artifacts: false
 
-build_docs_html:
+# Doc jobs have a lot of special cases, we specify rules here directly instead
+# in dependencies.yml to simplify things
+build_docs_html_full:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+
+build_docs_html_fast:
   extends:
     - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_full
+      when: never
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
   artifacts:
     when: always
     paths:
@@ -64,11 +127,14 @@ build_docs_html:
     expire_in: 4 days
   variables:
     DOC_BUILDERS: "html"
+    DOCS_FAST_BUILD: "yes"
 
 build_docs_pdf:
   extends:
     - .build_docs_build_stage_template
-    - .rules:build:docs:label-only
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
   artifacts:
     when: always
     paths:
@@ -81,6 +147,8 @@ build_docs_pdf:
   extends:
     - .before_script_no_sync_submodule
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.13
   stage: test_deploy
   tags:
     - deploy
@@ -92,16 +160,19 @@ build_docs_pdf:
   script:
     - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
     - export GIT_VER=$(git describe --always)
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
+    - pip install -r docs/requirements.txt
+    - deploy-docs
 
 # stage: test_deploy
 deploy_docs_preview:
   extends:
     - .deploy_docs_template
-    - .rules:build:docs:label-only-preview
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - build_docs_html
+  rules:
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+  dependencies:
+    - build_docs_html_fast
+    - build_docs_html_full
     - build_docs_pdf
   variables:
     TYPE: "preview"
@@ -117,11 +188,12 @@ deploy_docs_production:
   # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
   extends:
     - .deploy_docs_template
-    - .rules:build:docs-production
+  rules:
+    - <<: *if-protected-no_label
   stage: post_deploy
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
+    - build_docs_html_full
     - build_docs_pdf
     - job: push_to_github
       artifacts: false
@@ -148,6 +220,6 @@ check_doc_links:
   allow_failure: true
   script:
     - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - pip install -r requirements.txt
     # At the moment this check will always fail due to multiple known limitations, ignore result
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck || { echo "THERE ARE ISSUES DUE TO KNOWN LIMITATIONS, PLEASE FIX THEM. Nowadays we're ignored them to pass pipeline."; true; }
+    - build-docs -t $DOCTGT -l $DOCLANG linkcheck || { echo "THERE ARE ISSUES DUE TO KNOWN LIMITATIONS, PLEASE FIX THEM. Nowadays we're ignored them to pass pipeline."; true; }

.gitlab/ci/host-test.yml

@@ -1,7 +1,9 @@
 .host_test_template:
   extends: .rules:test:host_test
   stage: host_test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.13
   tags:
     - host_test
   dependencies: []
@@ -11,7 +13,7 @@ test_nvs_on_host:
   extends: .host_test_template
   script:
     - cd components/nvs_flash/test_nvs_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh make test
+    - make test
 
 test_nvs_coverage:
   extends:
@@ -33,7 +35,7 @@ test_partition_table_on_host:
     - build
   script:
     - cd components/partition_table/test_gen_esp32part_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./gen_esp32part_tests.py
+    - ./gen_esp32part_tests.py
 
 test_wl_on_host:
   extends: .host_test_template
@@ -55,10 +57,10 @@ test_ldgen_on_host:
   extends: .host_test_template
   script:
     - cd tools/ldgen/test
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_fragments.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_generation.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_entity.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_output_commands.py
+    - ./test_fragments.py
+    - ./test_generation.py
+    - ./test_entity.py
+    - ./test_output_commands.py
   variables:
     LC_ALL: C.UTF-8
 
@@ -66,7 +68,7 @@ test_ldgen_on_host:
   extends:
     - .host_test_template
     - .rules:labels:fuzzer_test-weekend_test
-  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+  image: $AFL_FUZZER_TEST_IMAGE
   artifacts:
     when: always
     paths:
@@ -110,6 +112,8 @@ test_spiffs_on_host:
   script:
     - cd components/spiffs/test_spiffs_host/
     - make test
+    - cd ../test_spiffsgen
+    - ./test_spiffsgen.py
 
 test_multi_heap_on_host:
   extends: .host_test_template
@@ -123,26 +127,26 @@ test_certificate_bundle_on_host:
     - build
   script:
     - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_gen_crt_bundle.py
+    - ./test_gen_crt_bundle.py
 
 test_confserver:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/confserver
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confserver.py
+    - ./test_confserver.py
 
 test_gen_kconfig_doc:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/gen_kconfig_doc/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_target_visibility.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_kconfig_out.py
+    - ./test_target_visibility.py
+    - ./test_kconfig_out.py
 
 test_confgen:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/confgen/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confgen.py
+    - ./test_confgen.py
 
 test_idf_monitor:
   extends: .host_test_template
@@ -166,7 +170,7 @@ test_idf_size:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/test_idf_size
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_idf_py:
   extends: .host_test_template
@@ -174,108 +178,80 @@ test_idf_py:
     LC_ALL: C.UTF-8
   script:
     - cd ${IDF_PATH}/tools/test_idf_py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_py.py
+    - ./test_idf_py.py
 
 test_idf_tools:
   extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/tools.new.json
+    expire_in: 1 week
   script:
     # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
     - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/root/.espressif/tools\|/opt/espressif" | tr "\n" ":"); echo ${p%:})
     - cd ${IDF_PATH}/tools/test_idf_tools
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+    - ./test_idf_tools.py
     # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
     - cd ${IDF_PATH}/tools
     - python3 ./idf_tools.py install-python-env
 
-test_esp_err_to_name_on_host:
-  extends: .host_test_template
-  artifacts:
-    when: on_failure
-    paths:
-      - components/esp32/esp_err_to_name.c
-    expire_in: 1 week
-  script:
-    - cd ${IDF_PATH}/tools/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./gen_esp_err_to_name.py
-    - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./gen_esp_err_to_name.py
-    - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
-
-test_esp32_efuse_table_on_host:
+.test_efuse_table_on_host_template:
   extends: .host_test_template
+  variables:
+    IDF_TARGET: "esp32"
   artifacts:
     when: on_failure
     paths:
-      - components/efuse/esp32/esp_efuse_table.c
+      - components/efuse/${IDF_TARGET}/esp_efuse_table.c
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
-    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found for esp32 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
-    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
+    - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv
+    - git diff --exit-code -- ${IDF_TARGET}/esp_efuse_table.c || { echo 'Differences found for ${IDF_TARGET} target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
-test_esp32s2_efuse_table_on_host:
-  extends: .host_test_template
-  artifacts:
-    when: on_failure
-    paths:
-      - components/efuse/esp32s2/esp_efuse_table.c
-    expire_in: 1 week
-  script:
-    - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s2/esp_efuse_table.c || { echo 'Differences found for esp32s2 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s2/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
-    - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+test_efuse_table_on_host_esp32:
+  extends: .test_efuse_table_on_host_template
 
-test_esp32s3_efuse_table_on_host:
-  extends: .host_test_template
-  artifacts:
-    when: on_failure
-    paths:
-      - components/efuse/esp32s3/esp_efuse_table.c
-    expire_in: 1 week
-  script:
-    - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s3/esp_efuse_table.c || { echo 'Differences found for esp32s3 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s3/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
-    - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+test_efuse_table_on_host_esp32s2:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32s2
 
-test_esp32c3_efuse_table_on_host:
-  extends: .host_test_template
-  artifacts:
-    when: on_failure
-    paths:
-      - components/efuse/esp32c3/esp_efuse_table.c
-    expire_in: 1 week
-  script:
-    - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32c3/esp_efuse_table.c || { echo 'Differences found for esp32c3 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32c3/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
-    - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+test_efuse_table_on_host_esp32s3:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32s3
+
+test_efuse_table_on_host_esp32c3:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32c3
+
+test_efuse_table_on_host_esp32h2:
+  extends: .test_efuse_table_on_host_template
+  variables:
+    IDF_TARGET: esp32h2
 
 test_espcoredump:
   extends: .host_test_template
   artifacts:
     when: always
     paths:
-      - components/espcoredump/test/.coverage
-      - components/espcoredump/test/output
+      - components/espcoredump/test/**/.coverage
+      - components/espcoredump/test/**/output
     expire_in: 1 week
+  variables:
+    IDF_COREDUMP_ELF_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/idf-coredump-elf.git"
+    IDF_COREDUMP_ELF_TAG: idf-20210910-00
+    # install CMake version specified in tools.json
+    SETUP_TOOLS_LIST: "all"
   script:
-    - cd components/espcoredump/test/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_espcoredump.sh
+    - retry_failed git clone ${IDF_COREDUMP_ELF_REPO} -b $IDF_COREDUMP_ELF_TAG
+    - cd ${IDF_PATH}/components/espcoredump/test/
+    - ./test_espcoredump.sh ${CI_PROJECT_DIR}/idf-coredump-elf
 
 test_logtrace_proc:
   extends: .host_test_template
@@ -287,7 +263,7 @@ test_logtrace_proc:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_sysviewtrace_proc:
   extends: .host_test_template
@@ -299,7 +275,7 @@ test_sysviewtrace_proc:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_mkdfu:
   extends: .host_test_template
@@ -307,27 +283,13 @@ test_mkdfu:
     LC_ALL: C.UTF-8
   script:
     - cd ${IDF_PATH}/tools/test_mkdfu
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py
+    - ./test_mkdfu.py
 
 test_mkuf2:
   extends: .host_test_template
   script:
     - cd ${IDF_PATH}/tools/test_mkuf2
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkuf2.py
-
-test_docs:
-  extends: .host_test_template
-  image: $ESP_IDF_DOC_ENV_IMAGE
-  artifacts:
-    when: on_failure
-    paths:
-      - docs/test/_build/*/*/*/html/*
-    expire_in: 1 week
-  script:
-    - cd ${IDF_PATH}/docs/test
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
+    - ./test_mkuf2.py
 
 test_autocomplete:
   extends: .host_test_template
@@ -338,7 +300,7 @@ test_autocomplete:
       - ${IDF_PATH}/*.out
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_autocomplete.py
+    - ${IDF_PATH}/tools/ci/test_autocomplete.py
 
 test_detect_python:
   extends: .host_test_template
@@ -358,4 +320,53 @@ test_nvs_page:
   script:
     - cd ${IDF_PATH}/components/nvs_flash/host_test/nvs_page_test
     - idf.py build
-    - build/host_nvs_page_test.elf
+    - build/test_nvs_page_host.elf
+
+test_log:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/log/host_test/log_test
+    - idf.py build
+    - build/test_log_host.elf
+
+test_esp_event:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_event/host_test/esp_event_unit_test
+    - idf.py build
+    - build/test_esp_event_host.elf
+
+test_esp_timer_cxx:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/esp_timer
+    - idf.py build
+    - build/test_esp_timer_cxx_host.elf
+
+test_eh_frame_parser:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_system/test_eh_frame_parser
+    - make
+    - ./eh_frame_test
+
+test_rom_on_linux_works:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_rom/host_test/rom_test
+    - idf.py build
+    - build/test_rom_host.elf
+
+test_cxx_gpio:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/examples/cxx/experimental/experimental_cpp_component/host_test/gpio
+    - idf.py build
+    - build/test_gpio_cxx_host.elf
+
+test_linux_example:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/examples/build_system/cmake/linux_host_app
+    - idf.py build
+    - build/linux_host_app.elf

.gitlab/ci/pre_check.yml

@@ -1,6 +1,6 @@
 .pre_check_base_template:
   stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - host_test
   dependencies: []
@@ -50,22 +50,26 @@ check_rom_api_header:
   extends: .pre_check_job_template
   script:
     - tools/ci/check_examples_rom_header.sh
-    - tools/ci/check_rom_apis.sh
+    - tools/ci/check_api_violation.sh
 
 check_python_style:
   extends:
     - .pre_check_base_template
     - .rules:patterns:python-files
+  variables:
+    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
       - flake8_output.txt
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+    - python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
 
 test_check_kconfigs:
   extends: .pre_check_job_template
+  variables:
+    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
@@ -77,7 +81,7 @@ test_check_kconfigs:
       - tools/*/*/*/Kconfig*.new
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_check_kconfigs.py
+    - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py
 
 check_wifi_lib_md5:
   extends: .pre_check_base_template
@@ -91,7 +95,7 @@ check_wifi_lib_md5:
 
 check_fuzzer_compilation:
    extends: .pre_check_base_template
-   image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+   image: $AFL_FUZZER_TEST_IMAGE
    script:
      - cd ${IDF_PATH}/components/lwip/test_afl_host
      - make MODE=dhcp_server
@@ -109,6 +113,31 @@ check_public_headers:
   script:
     - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
 
+check_soc_struct_headers:
+  extends:
+    - .pre_check_base_template
+    - .rules:build
+  tags:
+    - build
+  script:
+    - find ${IDF_PATH}/components/soc/*/include/soc/ -name "*_struct.h" -print0 | xargs -0 -n1 ./tools/ci/check_soc_struct_headers.py
+
+check_esp_err_to_name:
+  extends:
+    - .pre_check_base_template
+    - .rules:build
+  tags:
+    - build
+  artifacts:
+    when: on_failure
+    paths:
+      - components/esp_common/esp_err_to_name.c
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/
+    - ./gen_esp_err_to_name.py
+    - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
+
 scan_tests:
   extends:
     - .pre_check_base_template

.gitlab/ci/rules.yml

@@ -1,19 +1,12 @@
 ############
 # Patterns #
 ############
-.patterns-docs: &patterns-docs
-  - ".gitlab/ci/docs.yml"
-  - "docs/**/*"
-  - "components/**/*.h"
-  - "components/**/Kconfig*"
-  - "components/**/CMakeList.txt"
-  - "components/**/sdkconfig*"
-  - "tools/kconfig_new/**/*"
-  - "CONTRIBUTING.rst"
-
 .patterns-c-files: &patterns-c-files
   - ".gitlab/ci/static-code-analysis.yml"
+
   - "tools/ci/static-analysis-rules.yml"
+  - "tools/ci/clang_tidy_dirs.txt"
+
   - "**/*.{c,C}"
   - "**/*.{h,H}"
   - "components/**/Kconfig"
@@ -23,6 +16,9 @@
   - ".gitlab/ci/static-code-analysis.yml"
   - "**/*.py"
 
+.patterns-sonarqube-files: &patterns-sonarqube-files
+  - "tools/ci/sonar_exclude_list.txt"
+
 .patterns-example_test: &patterns-example_test
   - "tools/ci/python_packages/gitlab_api.py"
   - "tools/ci/python_packages/idf_http_server_test/**/*"
@@ -44,11 +40,13 @@
 
 .patterns-build_components: &patterns-build_components
   - "components/**/*"
+  - "examples/cxx/experimental/experimental_cpp_component/*"
 
 .patterns-build_system: &patterns-build_system
   - "tools/cmake/**/*"
   - "tools/kconfig_new/**/*"
   - "tools/tools.json"
+  - "tools/ci/test_build_system*.sh"
 
 .patterns-custom_test: &patterns-custom_test
   - "components/espcoredump/**/*"
@@ -94,6 +92,9 @@
 
 .patterns-host_test: &patterns-host_test
   - ".gitlab/ci/host-test.yml"
+
+  - "components/**/*"
+
   - "tools/ci/test_autocomplete.py"
   - "tools/ci/test_build_system.sh"
   - "tools/ci/test_build_system_cmake.sh"
@@ -101,12 +102,11 @@
   - "tools/ci/test_configure_ci_environment.sh"
 
   - "tools/mass_mfg/**/*"
-  - "components/nvs_flash/test_nvs_host/**/*"
 
   - "tools/esp_app_trace/**/*"
   - "tools/ldgen/**/*"
 
-  - "tools/gdb_panic_server.py"
+  - "tools/idf_monitor_base/*"
   - "tools/idf_monitor.py"
   - "tools/test_idf_monitor/**/*"
 
@@ -117,6 +117,7 @@
   - "tools/idf_size.py"
   - "tools/test_idf_size/**/*"
 
+  - "tools/tools.json"
   - "tools/tools_schema.json"
   - "tools/idf_tools.py"
   - "tools/test_idf_tools/**/*"
@@ -147,6 +148,7 @@
   - "components/cmock/CMock/vendor/unity"
   - "components/coap/libcoap"
   - "components/coap/libcoap/ext/tinydtls"
+  - "components/esp_phy/lib"
   - "components/esp_wifi/lib"
   - "components/esptool_py/esptool"
   - "components/expat/expat"
@@ -158,6 +160,7 @@
   - "components/nghttp/nghttp2"
   - "components/nghttp/nghttp2/third-party/mruby"
   - "components/nghttp/nghttp2/third-party/neverbleed"
+  - "components/openthread/lib"
   - "components/protobuf-c/protobuf-c"
   - "components/spiffs/spiffs"
   - "components/tinyusb/tinyusb"
@@ -242,6 +245,8 @@
       changes: *patterns-c-files
     - <<: *if-dev-push
       changes: *patterns-python-files
+    - <<: *if-dev-push
+      changes: *patterns-sonarqube-files
 
 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 # DO NOT place comments or maintain any code from this line
@@ -258,9 +263,6 @@
 .if-label-build: &if-label-build
   if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'
 
-.if-label-build_docs: &if-label-build_docs
-  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-component_ut: &if-label-component_ut
   if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i'
 
@@ -294,9 +296,6 @@
 .if-label-docker: &if-label-docker
   if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i'
 
-.if-label-docs: &if-label-docs
-  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-example_test: &if-label-example_test
   if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i'
 
@@ -432,6 +431,7 @@
     - <<: *if-label-build
     - <<: *if-label-custom_test
     - <<: *if-label-custom_test_esp32
+    - <<: *if-label-weekend_test
     - <<: *if-dev-push
       changes: *patterns-build_components
     - <<: *if-dev-push
@@ -486,29 +486,6 @@
     - <<: *if-dev-push
       changes: *patterns-docker
 
-.rules:build:docs:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-    - <<: *if-dev-push
-      changes: *patterns-docs
-
-.rules:build:docs-production:
-  rules:
-    - <<: *if-protected-no_label
-
-.rules:build:docs:label-only:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
-.rules:build:docs:label-only-preview:
-  rules:
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
 .rules:build:example_test-esp32:
   rules:
     - <<: *if-protected
@@ -516,7 +493,6 @@
     - <<: *if-label-example_test
     - <<: *if-label-example_test_esp32
     - <<: *if-label-iperf_stress_test
-    - <<: *if-label-weekend_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -612,11 +588,13 @@
     - <<: *if-label-example_test_esp32s2
     - <<: *if-label-example_test_esp32s3
     - <<: *if-label-integration_test
+    - <<: *if-label-iperf_stress_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32
     - <<: *if-label-unit_test_esp32c3
     - <<: *if-label-unit_test_esp32s2
     - <<: *if-label-unit_test_esp32s3
+    - <<: *if-label-weekend_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push
@@ -740,11 +718,13 @@
     - <<: *if-label-example_test_esp32s3
     - <<: *if-label-host_test
     - <<: *if-label-integration_test
+    - <<: *if-label-iperf_stress_test
     - <<: *if-label-unit_test
     - <<: *if-label-unit_test_esp32
     - <<: *if-label-unit_test_esp32c3
     - <<: *if-label-unit_test_esp32s2
     - <<: *if-label-unit_test_esp32s3
+    - <<: *if-label-weekend_test
     - <<: *if-dev-push
       changes: *patterns-build-example_test
     - <<: *if-dev-push

.gitlab/ci/static-code-analysis.yml

@@ -3,19 +3,48 @@ clang_tidy_check:
   extends:
     - .pre_check_base_template
     - .rules:patterns:clang_tidy
-  image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
+  image: ${CLANG_STATIC_ANALYSIS_IMAGE}
   artifacts:
-    reports:
-      junit: $IDF_PATH/output.xml
-    when: always
     paths:
-      - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
+      - $OUTPUT_DIR
+    when: always
     expire_in: 1 day
+  variables:
+    CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
+    CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
+    RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
+    OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports
   script:
-    - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
-    # Setup parameters of triggered/regular job
-    - export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
-    - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
+    - python -m pip install -U pip
+    - internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang
+    - export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])")
+    - dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs)
+    - run_cmd idf_clang ${dirs}
+      --output-path ${OUTPUT_DIR}
+      --limit-file ${RULES_FILE}
+      --xtensa-include-dir
+      --run-clang-tidy-py ${RUN_CLANG_TIDY_PY}
+
+check_pylint:
+  extends:
+    - .pre_check_base_template
+    - .rules:patterns:python-files
+    - .before_script_minimal
+  image: $SONARQUBE_SCANNER_IMAGE
+  artifacts:
+    when: always
+    paths:
+      - pylint-report.txt
+    expire_in: 1 week
+  script:
+    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+    - |
+      if [ -n "$CI_MERGE_REQUEST_IID" ]; then
+        export files=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | grep ".py");
+      else
+        export files=$(find . -iname "*.py" -print);
+      fi
+    - pylint --rcfile=.pylintrc $files -r n --output-format=parseable > pylint-report.txt || exit 0
 
 # build stage
 # Sonarqube related jobs put here for this reason:
@@ -33,26 +62,31 @@ clang_tidy_check:
 .sonar_scan_template:
   stage: build
   image:
-    name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2
+    name: $SONARQUBE_SCANNER_IMAGE
   before_script:
     - source tools/ci/utils.sh
     - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
     - fetch_submodules
     # Exclude the submodules, all paths ends with /**
-    - export SUBMODULES=$(get_all_submodules)
+    - submodules=$(get_all_submodules)
     # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
-    - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-    # Exclude the report dir
-    - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
-    - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
+    - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
+    # Exclude the report dir as well
+    - export EXCLUSIONS="$custom_excludes,$submodules"
+    - export SONAR_SCANNER_OPTS="-Xmx2048m"
+
   variables:
     GIT_DEPTH: 0
-    NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py
-    REPORT_DIR: examples/get-started/hello_world/tidybuild/report
+    REPORT_PATTERN: clang_tidy_reports/*.txt
+  artifacts:
+    when: always
+    paths:
+      - $REPORT_PATTERN
   tags:
     - host_test
   dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
     - clang_tidy_check
+    - check_pylint
 
 code_quality_check:
   extends:
@@ -67,23 +101,16 @@ code_quality_check:
     - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
     - sonar-scanner
       -Dsonar.analysis.mode=preview
-      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
-      -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
-      -Dsonar.cxx.includeDirectories=components,/usr/include
+      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
       -Dsonar.exclusions=$EXCLUSIONS
       -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
       -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
-      -Dsonar.gitlab.failure_notification_mode=exit-code
       -Dsonar.gitlab.merge_request_discussion=true
-      -Dsonar.gitlab.project_id=$CI_PROJECT_ID
-      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
+      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
       -Dsonar.host.url=$SONAR_HOST_URL
       -Dsonar.login=$SONAR_LOGIN
-      -Dsonar.projectBaseDir=$CI_PROJECT_DIR
-      -Dsonar.projectKey=esp-idf
-      -Dsonar.python.pylint_config=.pylintrc
-      -Dsonar.sourceEncoding=UTF-8
-      -Dsonar.sources=$CI_PROJECT_DIR
+      -Dsonar.python.pylint.reportPath=pylint-report.txt
 
 code_quality_report:
   extends:
@@ -94,42 +121,10 @@ code_quality_report:
   script:
     - sonar-scanner
       -Dsonar.branch.name=$CI_COMMIT_REF_NAME
-      -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
-      -Dsonar.cxx.includeDirectories=components,/usr/include
+      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
       -Dsonar.exclusions=$EXCLUSIONS
       -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA
-      -Dsonar.gitlab.failure_notification_mode=exit-code
-      -Dsonar.gitlab.project_id=$CI_PROJECT_ID
       -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
       -Dsonar.host.url=$SONAR_HOST_URL
       -Dsonar.login=$SONAR_LOGIN
-      -Dsonar.projectBaseDir=$CI_PROJECT_DIR
-      -Dsonar.projectKey=esp-idf
-      -Dsonar.python.pylint_config=.pylintrc
-      -Dsonar.sourceEncoding=UTF-8
-      -Dsonar.sources=$CI_PROJECT_DIR
-
-# deploy stage
-clang_tidy_deploy:
-  extends:
-    - .deploy_job_template
-    - .rules:patterns:clang_tidy
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - clang_tidy_check
-  tags:
-    - deploy
-    - shiny
-  script:
-    - add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER
-    - export GIT_VER=$(git describe --always)
-    - cd $IDF_PATH/examples/get-started/hello_world/tidybuild
-    - mv report $GIT_VER
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
-    - export STATIC_REPORT_PATH="web/static_analysis/esp-idf/"
-    - ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy"
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy
-    - ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
-    # add link to view the report
-    - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
-    - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }
+      -Dsonar.python.pylint.reportPath=pylint-report.txt

.gitlab/ci/target-test.yml

@@ -12,6 +12,7 @@
   artifacts:
     when: always
     paths:
+      - "**/*.log"
       - $LOG_PATH
     expire_in: 1 week
     reports:
@@ -45,33 +46,21 @@
   extends:
     - .example_test_template
     - .rules:test:example_test-esp32
-variables:
+  variables:
     SUBMODULES_TO_FETCH: "all"
 
 test_weekend_mqtt:
   extends:
-    - .example_test_template
+    - .test_app_esp32_template
     - .rules:labels:weekend_test
   tags:
     - ESP32
     - Example_WIFI
-  variables:
-    ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
-    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
-
-test_weekend_network:
-  extends:
-    - .example_test_template
-    - .rules:labels:weekend_test
-  image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - ESP32
-    - Example_WIFI
-  variables:
-    ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
-    CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
+  script:
+    - export MQTT_PUBLISH_TEST=1
+    - export TEST_PATH=$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test
+    - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin
+    - run_cmd python Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_.yml
 
 .example_test_esp32_template:
   extends:
@@ -108,7 +97,7 @@ example_test_001B_V3:
 
 example_test_001C:
   extends: .example_test_esp32_template
-  parallel: 3
+  parallel: 4
   tags:
     - ESP32
     - Example_GENERIC
@@ -140,6 +129,12 @@ example_test_002:
     - ESP32
     - Example_ShieldBox_Basic
 
+example_test_ethernet:
+  extends: .example_test_esp32_template
+  tags:
+    - ESP32
+    - Example_Ethernet
+
 .example_test_003:
   extends: .example_test_esp32_template
   tags:
@@ -196,9 +191,6 @@ example_test_009:
   tags:
     - ESP32
     - test_jtag_arm
-  artifacts:
-    paths:
-      - $CI_PROJECT_DIR/examples/*/*/*.log
   variables:
     SETUP_TOOLS: "1"
     PYTHON_VER: 3
@@ -214,10 +206,6 @@ example_test_011:
   tags:
     - ESP32
     - Example_T2_RS485
-  artifacts:
-    paths:
-      - $CI_PROJECT_DIR/examples/protocols/modbus/serial/*.log
-      - $LOG_PATH
   variables:
     SETUP_TOOLS: "1"
 
@@ -244,20 +232,12 @@ example_test_015:
   tags:
     - ESP32
     - Example_PPP
-  artifacts:
-    paths:
-      - $CI_PROJECT_DIR/examples/*/*/*.log
-      - $LOG_PATH
 
 example_test_016:
   extends: .example_test_esp32_template
   tags:
     - ESP32
     - Example_Modbus_TCP
-  artifacts:
-    paths:
-      - $CI_PROJECT_DIR/examples/*/*/*.log
-      - $LOG_PATH
 
 example_test_017:
   extends: .example_test_esp32s2_template
@@ -290,20 +270,17 @@ example_test_ESP32_SDSPI:
     - ESP32
     - UT_T1_SPIMODE
 
-# uncomment when ESP32S2 & ESP32C3 runners with external SD connected over SPI are available
-# ensure the runners have required tags created
-#
-#example_test_ESP32S2_SDSPI:
-#  extends: .example_test_esp32s2_template
-#  tags:
-#    - ESP32S2
-#    - UT_T1_SPIMODE
-#
-#example_test_ESP32C3_SDSPI:
-#  extends: .example_test_esp32c3_template
-#  tags:
-#    - ESP32C3
-#    - UT_T1_SPIMODE
+example_test_ESP32S2_SDSPI:
+  extends: .example_test_esp32s2_template
+  tags:
+    - ESP32S2
+    - UT_T1_SPIMODE
+
+example_test_ESP32C3_SDSPI:
+  extends: .example_test_esp32c3_template
+  tags:
+    - ESP32C3
+    - UT_T1_SPIMODE
 
 .test_app_template:
   extends: .target_test_job_template
@@ -331,10 +308,6 @@ test_app_test_001:
   tags:
     - ESP32
     - test_jtag_arm
-  artifacts:
-    paths:
-      - $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
-      - $LOG_PATH/*/XUNIT_RESULT.xml
   variables:
     SETUP_TOOLS: "1"
 
@@ -395,12 +368,45 @@ test_app_test_esp32_generic:
     - .component_ut_template
     - .rules:test:component_ut-esp32
 
+.component_ut_esp32s2_template:
+  extends:
+    - .component_ut_template
+    - .rules:test:component_ut-esp32s2
+
+.component_ut_esp32s3_template:
+  extends:
+    - .component_ut_template
+    - .rules:test:component_ut-esp32s3
+
+.component_ut_esp32c3_template:
+  extends:
+    - .component_ut_template
+    - .rules:test:component_ut-esp32c3
+
 component_ut_test_001:
   extends: .component_ut_esp32_template
   tags:
     - ESP32
     - COMPONENT_UT_GENERIC
 
+component_ut_test_esp32s2:
+  extends: .component_ut_esp32s2_template
+  tags:
+    - ESP32S2
+    - COMPONENT_UT_GENERIC
+
+component_ut_test_esp32s3:
+  extends: .component_ut_esp32s3_template
+  tags:
+    - ESP32S3
+    - COMPONENT_UT_GENERIC
+
+component_ut_test_esp32c3:
+  extends: .component_ut_esp32c3_template
+  tags:
+    - ESP32C3
+    - COMPONENT_UT_GENERIC
+
 .unit_test_template:
   extends: .target_test_job_template
   variables:
@@ -417,6 +423,11 @@ component_ut_test_001:
     - .unit_test_template
     - .rules:test:unit_test-esp32s2
 
+.unit_test_esp32s3_template:
+  extends:
+    - .unit_test_template
+    - .rules:test:unit_test-esp32s3
+
 .unit_test_esp32c3_template:
   extends:
     - .unit_test_template
@@ -424,7 +435,7 @@ component_ut_test_001:
 
 UT_001:
   extends: .unit_test_esp32_template
-  parallel: 49
+  parallel: 50
   tags:
     - ESP32_IDF
     - UT_T1_1
@@ -433,7 +444,7 @@ UT_001:
 
 UT_002:
   extends: .unit_test_esp32_template
-  parallel: 15
+  parallel: 14
   tags:
     - ESP32_IDF
     - UT_T1_1
@@ -466,6 +477,13 @@ UT_006:
     - UT_T1_SPIMODE
     - psram
 
+UT_007:
+  extends: .unit_test_esp32_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
 UT_008:
   extends: .unit_test_esp32_template
   tags:
@@ -487,19 +505,6 @@ UT_014:
     - UT_T2_RS485
     - psram
 
-UT_015:
-  extends: .unit_test_esp32_template
-  tags:
-    - ESP32_IDF
-    - UT_T1_RMT
-
-UT_016:
-  extends: .unit_test_esp32_template
-  tags:
-    - ESP32_IDF
-    - UT_T1_RMT
-    - psram
-
 UT_017:
   extends: .unit_test_esp32_template
   tags:
@@ -523,7 +528,6 @@ UT_020:
 
 UT_021:
   extends: .unit_test_esp32_template
-  parallel: 2
   tags:
     - ESP32_IDF
     - psram
@@ -536,20 +540,6 @@ UT_022:
     - UT_T2_I2C
     - psram
 
-UT_023:
-  extends: .unit_test_esp32_template
-  parallel: 2
-  tags:
-    - ESP32_IDF
-    - UT_T1_MCPWM
-
-UT_024:
-  extends: .unit_test_esp32_template
-  tags:
-    - ESP32_IDF
-    - UT_T1_MCPWM
-    - psram
-
 UT_028:
   extends: .unit_test_esp32_template
   tags:
@@ -572,7 +562,7 @@ UT_033:
 
 UT_034:
   extends: .unit_test_esp32_template
-  parallel: 3
+  parallel: 2
   tags:
     - ESP32_IDF
     - UT_T1_ESP_FLASH
@@ -600,7 +590,7 @@ UT_036:
 
 UT_038:
   extends: .unit_test_esp32s2_template
-  parallel: 3
+  parallel: 2
   tags:
     - ESP32S2_IDF
     - UT_T1_ESP_FLASH
@@ -640,14 +630,26 @@ UT_046:
 
 UT_047:
   extends: .unit_test_esp32s2_template
-  parallel: 3
+  parallel: 5
   tags:
     - ESP32S2_IDF
     - UT_T1_1
 
+UT_S2_SPI_DUAL:
+  extends: .unit_test_esp32s2_template
+  tags:
+    - ESP32S2_IDF
+    - Example_SPI_Multi_device
+
+UT_S2_SDSPI:
+  extends: .unit_test_esp32s2_template
+  tags:
+    - ESP32S2_IDF
+    - UT_T1_SPIMODE
+
 UT_C3:
   extends: .unit_test_esp32c3_template
-  parallel: 31
+  parallel: 32
   tags:
     - ESP32C3_IDF
     - UT_T1_1
@@ -661,7 +663,6 @@ UT_C3_FLASH:
 
 UT_C3_SPI_DUAL:
   extends: .unit_test_esp32c3_template
-  parallel: 2
   tags:
     - ESP32C3_IDF
     - Example_SPI_Multi_device
@@ -684,6 +685,44 @@ UT_C3_FLASH_SUSPEND:
     - ESP32C3_IDF
     - UT_T1_Flash_Suspend
 
+UT_C3_SDSPI:
+  extends: .unit_test_esp32c3_template
+  tags:
+    - ESP32C3_IDF
+    - UT_T1_SPIMODE
+
+UT_S3:
+  extends: .unit_test_esp32s3_template
+  parallel: 29
+  tags:
+    - ESP32S3_IDF
+    - UT_T1_1
+
+UT_S3_SPI_DUAL:
+  extends: .unit_test_esp32s3_template
+  tags:
+    - ESP32S3_IDF
+    - Example_SPI_Multi_device
+
+UT_S3_FLASH:
+  extends: .unit_test_esp32s3_template
+  parallel: 2
+  tags:
+    - ESP32S3_IDF
+    - UT_T1_ESP_FLASH
+
+component_ut_test_ip101:
+  extends: .component_ut_esp32_template
+  tags:
+    - ESP32
+    - COMPONENT_UT_IP101
+
+component_ut_test_lan8720:
+  extends: .component_ut_esp32_template
+  tags:
+    - ESP32
+    - COMPONENT_UT_LAN8720
+
 .integration_test_template:
   extends:
     - .target_test_job_template
@@ -693,7 +732,7 @@ UT_C3_FLASH_SUSPEND:
     - build_ssc_esp32
   variables:
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
-    LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
+    LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
     MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"

.gitlab/merge_request_templates/Mixed Template.md

@@ -17,7 +17,6 @@ _This entire section can be deleted if all items are checked._
 * [ ] Release note entry if this is a new public feature, or a fix for an issue introduced in the previous release.
 * [ ] The commit log is clean and ready to merge.
 * [ ] All relevant CI jobs have been run, i.e. jobs which cover the code changed by the MR.
-* [ ] Pipeline with Python 3 has been successfully run, i.e. `@bot ...; with Python3`, if the MR adds or changes Python code.
 
 ---
 

.gitmodules

@@ -98,3 +98,15 @@
 [submodule "components/bt/controller/lib_esp32c3_family"]
 	path = components/bt/controller/lib_esp32c3_family
 	url = ../../espressif/esp32c3-bt-lib.git
+
+[submodule "components/esp_phy/lib"]
+	path = components/esp_phy/lib
+	url = ../../espressif/esp-phy-lib.git
+
+[submodule "components/openthread/lib"]
+	path = components/openthread/lib
+	url = ../../espressif/esp-thread-lib.git
+
+[submodule "components/ieee802154/lib"]
+	path = components/ieee802154/lib
+	url = ../../espressif/esp-ieee802154-lib.git

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.0.1
     hooks:
       - id: trailing-whitespace
         # note: whitespace exclusions use multiline regex, see https://pre-commit.com/#regular-expressions
@@ -26,12 +26,12 @@ repos:
         args: ['-f=lf']
       - id: double-quote-string-fixer
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.2
     hooks:
       - id: flake8
         args: ['--config=.flake8', '--tee', '--benchmark']
   - repo: https://github.com/pycqa/isort
-    rev: 5.6.4
+    rev: 5.9.3
     hooks:
       - id: isort
         name: isort (python)
@@ -92,11 +92,24 @@ repos:
       - id: mypy-check
         name: Check type annotations in python files
         entry: tools/ci/check_type_comments.py
-        additional_dependencies: ['mypy==0.800', 'mypy-extensions==0.4.3']
+        additional_dependencies:
+          - 'mypy==0.800'
+          - 'mypy-extensions==0.4.3'
         language: python
         types: [python]
+      - id: check-copyright
+        name: Check copyright notices
+        entry: tools/ci/check_copyright.py --verbose --replace
+        additional_dependencies:
+          - 'comment_parser == 1.2.3'
+          - 'thefuzz == 0.19.0'
+          - 'thefuzz[speedup] == 0.19.0; sys_platform != "win32"'
+          # don't depend on python-Levenshtein on Windows, as it requires Microsoft C++ Build Tools to install
+        language: python
+        files: \.(py|c|h|cpp|hpp|ld)$
+        require_serial: true
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.0.1
     hooks:
       - id: file-contents-sorter
-        files: '(tools\/ci\/executable-list\.txt|tools\/ci\/mypy_ignore_list\.txt)'
+        files: 'tools\/ci\/(executable-list\.txt|mypy_ignore_list\.txt|check_copyright_ignore\.txt)'

CMakeLists.txt

@@ -20,7 +20,9 @@ if(NOT BOOTLOADER_BUILD)
 
     if(CONFIG_COMPILER_OPTIMIZATION_SIZE)
         list(APPEND compile_options "-Os")
-        list(APPEND compile_options "-freorder-blocks")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+            list(APPEND compile_options "-freorder-blocks")
+        endif()
     elseif(CONFIG_COMPILER_OPTIMIZATION_DEFAULT)
         list(APPEND compile_options "-Og")
     elseif(CONFIG_COMPILER_OPTIMIZATION_NONE)
@@ -33,7 +35,9 @@ else()  # BOOTLOADER_BUILD
 
     if(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE)
         list(APPEND compile_options "-Os")
-        list(APPEND compile_options "-freorder-blocks")
+        if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+            list(APPEND compile_options "-freorder-blocks")
+        endif()
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_DEBUG)
         list(APPEND compile_options "-Og")
     elseif(CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_NONE)
@@ -74,6 +78,52 @@ if(CONFIG_COMPILER_DISABLE_GCC8_WARNINGS)
                                 "-Wno-int-in-bool-context")
 endif()
 
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    list(APPEND c_compile_options "-Wno-old-style-declaration")
+endif()
+
+# Clang finds some warnings in IDF code which GCC doesn't.
+# All these warnings should be fixed before Clang is presented
+# as a toolchain choice for users.
+if(CMAKE_C_COMPILER_ID MATCHES "Clang")
+    # Clang checks Doxygen comments for being in sync with function prototype.
+    # There are some inconsistencies, especially in ROM headers.
+    list(APPEND compile_options "-Wno-documentation")
+    # GCC allows repeated typedefs when the source and target types are the same.
+    # Clang doesn't allow this. This occurs in many components due to forward
+    # declarations.
+    list(APPEND compile_options "-Wno-typedef-redefinition")
+    # This issue is seemingly related to newlib's char type functions.
+    # Fix is not clear yet.
+    list(APPEND compile_options "-Wno-char-subscripts")
+    # Clang seems to notice format string issues which GCC doesn't.
+    list(APPEND compile_options "-Wno-format-security")
+    # Logic bug in essl component
+    list(APPEND compile_options "-Wno-tautological-overlap-compare")
+    # Some pointer checks in mDNS component check addresses which can't be NULL
+    list(APPEND compile_options "-Wno-tautological-pointer-compare")
+    # Similar to the above, in tcp_transport
+    list(APPEND compile_options "-Wno-pointer-bool-conversion")
+    # mbedTLS md5.c triggers this warning in md5_test_buf (false positive)
+    list(APPEND compile_options "-Wno-string-concatenation")
+    # multiple cases of implict convertions between unrelated enum types
+    list(APPEND compile_options "-Wno-enum-conversion")
+    # When IRAM_ATTR is specified both in function declaration and definition,
+    # it produces different section names, since section names include __COUNTER__.
+    # Occurs in multiple places.
+    list(APPEND compile_options "-Wno-section")
+    # Multiple cases of attributes unknown to clang, for example
+    # __attribute__((optimize("-O3")))
+    list(APPEND compile_options "-Wno-unknown-attributes")
+    # Clang also produces many -Wunused-function warnings which GCC doesn't.
+    # However these aren't treated as errors.
+endif()
+# More warnings may exist in unit tests and example projects.
+
+if(CONFIG_COMPILER_WARN_WRITE_STRINGS)
+    list(APPEND compile_options "-Wwrite-strings")
+endif()
+
 if(CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE)
     list(APPEND compile_definitions "-DNDEBUG")
 endif()
@@ -104,8 +154,26 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
                                 )
 endif()
 
+if(CONFIG_ESP_SYSTEM_USE_EH_FRAME)
+  list(APPEND compile_options "-fasynchronous-unwind-tables")
+  list(APPEND link_options "-Wl,--eh-frame-hdr")
+endif()
+
 list(APPEND link_options "-fno-lto")
 
+# Placing jump tables in flash would cause issues with code that required
+# to be placed in IRAM
+list(APPEND compile_options "-fno-jump-tables")
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    # This flag is GCC-specific.
+    # Not clear yet if some other flag should be used for Clang.
+    list(APPEND compile_options "-fno-tree-switch-conversion")
+endif()
+
+if(CMAKE_C_COMPILER_ID MATCHES "LLVM")
+    list(APPEND compile_options "-fno-use-cxa-atexit")
+endif()
+
 idf_build_set_property(COMPILE_OPTIONS "${compile_options}" APPEND)
 idf_build_set_property(C_COMPILE_OPTIONS "${c_compile_options}" APPEND)
 idf_build_set_property(CXX_COMPILE_OPTIONS "${cxx_compile_options}" APPEND)

CONTRIBUTING.rst

@@ -56,7 +56,6 @@ Related Documents
     style-guide
     install-pre-commit-hook
     documenting-code
-    add-ons-reference
     creating-examples
     ../api-reference/template
     contributor-agreement

Kconfig

@@ -45,31 +45,29 @@ mainmenu "Espressif IoT Development Framework Configuration"
         default "y" if IDF_TARGET="esp32s3"
         select IDF_TARGET_ARCH_XTENSA
 
-    choice IDF_TARGET_ESP32S3_BETA_VERSION
-        prompt "ESP32-S3 beta version"
-        depends on IDF_TARGET_ESP32S3
-        default IDF_TARGET_ESP32S3_BETA_VERSION_3
-        help
-            Currently ESP32-S3 has several beta versions for internal use only.
-            Select the one that matches your chip model.
-
-        config IDF_TARGET_ESP32S3_BETA_VERSION_3
-            bool
-            prompt "ESP32-S3 beta3"
-    endchoice
-
     config IDF_TARGET_ESP32C3
         bool
         default "y" if IDF_TARGET="esp32c3"
         select FREERTOS_UNICORE
         select IDF_TARGET_ARCH_RISCV
 
+    config IDF_TARGET_ESP32H2
+        bool
+        default "y" if IDF_TARGET="esp32h2"
+        select FREERTOS_UNICORE
+        select IDF_TARGET_ARCH_RISCV
+
+    config IDF_TARGET_LINUX
+        bool
+        default "y" if IDF_TARGET="linux"
+
     config IDF_FIRMWARE_CHIP_ID
         hex
         default 0x0000 if IDF_TARGET_ESP32
         default 0x0002 if IDF_TARGET_ESP32S2
         default 0x0005 if IDF_TARGET_ESP32C3
-        default 0x0006 if IDF_TARGET_ESP32S3
+        default 0x0009 if IDF_TARGET_ESP32S3
+        default 0x000A if IDF_TARGET_ESP32H2 # ESP32H2-TODO: IDF-3475
         default 0xFFFF
 
     menu "SDK tool configuration"
@@ -79,6 +77,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
             default "xtensa-esp32s2-elf-" if IDF_TARGET_ESP32S2
             default "xtensa-esp32s3-elf-" if IDF_TARGET_ESP32S3
             default "riscv32-esp-elf-" if IDF_TARGET_ESP32C3
+            default "riscv32-esp-elf-" if IDF_TARGET_ESP32H2
 
             help
                 The prefix/path that is used to call the toolchain. The default setting assumes
@@ -154,7 +153,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
                     mon reset halt
                     # Run to a specific point in ROM code,
                     #  where most of initialization is complete.
-                    thb *0x40007901
+                    thb *0x40007d54
                     c
                     # Load the application into RAM
                     load
@@ -279,6 +278,12 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
         endchoice # assertions
 
+        config COMPILER_OPTIMIZATION_ASSERTION_LEVEL
+            int
+            default 0 if COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE
+            default 1 if COMPILER_OPTIMIZATION_ASSERTIONS_SILENT
+            default 2 if COMPILER_OPTIMIZATION_ASSERTIONS_ENABLE
+
         config COMPILER_OPTIMIZATION_CHECKS_SILENT
             bool "Disable messages in ESP_RETURN_ON_* and ESP_EXIT_ON_* macros"
             default n
@@ -357,6 +362,7 @@ mainmenu "Espressif IoT Development Framework Configuration"
 
                 - coverage: NORMAL < STRONG < OVERALL
 
+                The performance impact includes increasing the amount of stack memory required for each task.
 
             config COMPILER_STACK_CHECK_MODE_NONE
                 bool "None"

README.md

@@ -2,21 +2,32 @@
 
 * [中文版](./README_CN.md)
 
-ESP-IDF is the official development framework for the **ESP32** and **ESP32-S** Series SoCs provided for Windows, Linux and macOS.
+ESP-IDF is the development framework for Espressif SoCs supported on Windows, Linux and macOS.
+
+# ESP-IDF Release and SoC Compatibility
+
+The following table shows ESP-IDF support of Espressif SoCs where ![alt text][preview] and ![alt text][supported] denote preview status and support, respectively. In preview status the build is not yet enabled and some crucial parts could be missing (like documentation, datasheet). Please use an ESP-IDF release where the desired SoC is already supported.
+
+|Chip         |         v3.3           |          v4.0          |           v4.1         |          v4.2          |         v4.3           |          v4.4          |                                                            |
+|:----------- |:---------------------: | :---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:---------------------------------------------------------- |
+|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-S2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-C3     |                        |                        |                        |                        | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-S3     |                        |                        |                        |                        | ![alt text][preview]   | ![alt text][supported] | [Announcement](https://www.espressif.com/en/news/ESP32_S3) |
+|ESP32-H2     |                        |                        |                        |                        |                        | ![alt text][preview]   | [Announcement](https://www.espressif.com/en/news/ESP32_H2) |
+
+[supported]: https://img.shields.io/badge/-supported-green "supported"
+[preview]: https://img.shields.io/badge/-preview-orange "preview"
+
+Espressif SoCs released before 2016 (ESP8266 and ESP8285) are supported by [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK) instead.
 
 # Developing With ESP-IDF
 
 ## Setting Up ESP-IDF
 
-See setup guides for detailed instructions to set up the ESP-IDF:
-
-| Chip | Getting Started Guides for ESP-IDF |
-|:----:|:----|
-| <img src="docs/_static/chip-esp32.svg" height="90" alt="ESP32"> |  <ul><li>[stable](https://docs.espressif.com/projects/esp-idf/en/stable/esp32/) version</li><li>[latest (master branch)](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/) version</li></ul> |
-| <img src="docs/_static/chip-esp32-s2.svg" height="105" alt="ESP32-S2"> | <ul><li>[stable](https://docs.espressif.com/projects/esp-idf/en/stable/esp32s2/get-started/) version</li><li>[latest (master branch)](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/get-started/) version</li></ul> |
-| <img src="docs/_static/chip-esp32-c3.svg" height="75" alt="ESP32-C3"> | <ul><li>[latest (master branch)](https://docs.espressif.com/projects/esp-idf/en/latest/esp32c3/get-started/) version</li></ul> |
+See https://idf.espressif.com/ for links to detailed instructions on how to set up the ESP-IDF depending on chip you use.
 
-**Note:** Each ESP-IDF release has its own documentation. Please see Section [Versions](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/versions.html) how to find documentation and how to checkout specific release of ESP-IDF.
+**Note:** Each SoC series and each ESP-IDF release has its own documentation. Please see Section [Versions](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/versions.html) on how to find documentation and how to checkout specific release of ESP-IDF.
 
 ### Non-GitHub forks
 
@@ -71,7 +82,7 @@ You don't need to run `idf.py build` before running `idf.py flash`, `idf.py flas
 
 ## Viewing Serial Output
 
-The `idf.py monitor` target uses the [idf_monitor tool](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html) to display serial output from ESP32 or ESP32-S Series SoCs. idf_monitor also has a range of features to decode crash output and interact with the device. [Check the documentation page for details](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html).
+The `idf.py monitor` target uses the [idf_monitor tool](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html) to display serial output from Espressif SoCs. idf_monitor also has a range of features to decode crash output and interact with the device. [Check the documentation page for details](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html).
 
 Exit the monitor by typing Ctrl-].
 
@@ -105,5 +116,3 @@ This can be combined with other targets, ie `idf.py -p PORT erase_flash flash` w
 * [Check the Issues section on github](https://github.com/espressif/esp-idf/issues) if you find a bug or have a feature request. Please check existing Issues before opening a new one.
 
 * If you're interested in contributing to ESP-IDF, please check the [Contributions Guide](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/index.html).
-
-

README_CN.md

@@ -2,27 +2,39 @@
 
 * [English Version](./README.md)
 
-ESP-IDF 是由乐鑫官方推出的针对 **ESP32** 和 **ESP32-S2** 系列芯片的开发框架。
+ESP-IDF 是乐鑫官方推出的物联网开发框架，支持 Windows、Linux 和 macOS 操作系统。
 
-# 使用 ESP-IDF 进行开发
+# ESP-IDF 与乐鑫芯片
 
-## 搭建 ESP-IDF 开发环境
+下表总结了乐鑫芯片在 ESP-IDF 各版本中的支持状态，其中 ![alt text][supported] 代表已支持，![alt text][preview] 代表目前处于预览支持状态。在预览支持阶段，因为新芯片尚未完全添加到构建系统目录，所以一些重要的内容（如文档和技术规格书等）可能会缺失。请确保使用与芯片相匹配的 ESP-IDF 版本。
+
+|    芯片     |         v3.3           |          v4.0          |           v4.1         |          v4.2          |         v4.3           |          v4.4          |                                                            |
+|:----------- |:---------------------: | :---------------------:| :---------------------:| :---------------------:| :---------------------:| :---------------------:|:---------------------------------------------------------- |
+|ESP32        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-S2     |                        |                        |                        | ![alt text][supported] | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-C3     |                        |                        |                        |                        | ![alt text][supported] | ![alt text][supported] |                                                            |
+|ESP32-S3     |                        |                        |                        |                        | ![alt text][preview]   | ![alt text][supported] | [芯片发布公告](https://www.espressif.com/en/news/ESP32_S3) |
+|ESP32-H2     |                        |                        |                        |                        |                        | ![alt text][preview]   | [芯片发布公告](https://www.espressif.com/en/news/ESP32_H2) |
+
+[supported]: https://img.shields.io/badge/-%E6%94%AF%E6%8C%81-green "supported"
+[preview]: https://img.shields.io/badge/-%E9%A2%84%E8%A7%88-orange "preview"
 
-请参阅如下指南搭建 ESP-IDF 的开发环境：
+对于 2016 年之前发布的乐鑫芯片（包括 ESP8266 和 ESP8285），请参考 [RTOS SDK](https://github.com/espressif/ESP8266_RTOS_SDK)。
 
-| 芯片 | ESP-IDF 入门指南 |
-|:----:|:----|
-| <img src="docs/_static/chip-esp32.svg" height="90" alt="ESP32"> |  <ul><li>[稳定](https://docs.espressif.com/projects/esp-idf/en/stable/esp32/) 版本</li><li>[最新（master 分支)](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/) 版本</li></ul> |
-| <img src="docs/_static/chip-esp32-s2.svg" height="105" alt="ESP32-S2"> | <ul><li>[稳定](https://docs.espressif.com/projects/esp-idf/en/stable/esp32s2/get-started/) 版本</li><li>[最新（master 分支)](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/get-started/) 版本</li></ul> |
-| <img src="docs/_static/chip-esp32-c3.svg" height="75" alt="ESP32-C3"> | <ul><li>[最新（master 分支)](https://docs.espressif.com/projects/esp-idf/en/latest/esp32c3/get-started/) 版本</li></ul> |
+# 使用 ESP-IDF 进行开发
+
+## 搭建 ESP-IDF 开发环境
 
-**注意：** 每个 ESP-IDF 版本都有其对应的文档。 请参阅 [版本](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html) 部分，如何查找文档以及如何检出ESP-IDF的特定发行版。
+关于不同芯片如何搭建 ESP-IDF 的开发环境，请参考 https://idf.espressif.com/ 。
 
+**注意：** 不同系列芯片和不同 ESP-IDF 版本都有其对应的文档。请参阅[版本](https://docs.espressif.com/projects/esp-idf/zh_CN/latest/esp32/versions.html)部分，获得关于如何查找文档以及如何检出 ESP-IDF 的特定发行版的详细信息。
 
 ### 非 GitHub 分叉的 ESP-IDF 项目
 
 ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodules)），所以它们会指向 GitHub。
-如果 ESP-IDF 被分叉到的仓库不在 GitHub 上，那么你需要在克隆结束后运行该[脚本](tools/set-submodules-to-github.sh)。它会为所有的子模块设置绝对路径，接着可以通过 `git submodule update --init --recursive` 完成子模块的更新。
+如果 ESP-IDF 被分叉到的仓库不在 GitHub 上，那么你需要在克隆结束后运行该脚本 [tools/set-submodules-to-github.sh](tools/set-submodules-to-github.sh)。
+
+这个脚本会为所有的子模块设置绝对路径，接着可以通过 `git submodule update --init --recursive` 完成子模块的更新。
 如果 ESP-IDF 是从 GitHub 上克隆得到，则不需要此步骤。
 
 ## 寻找项目
@@ -39,24 +51,16 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 
 ## 设置构建环境
 
-（请参考入门指南中列出的详细步骤。）
-* 在主机中安装入门指南中提到的构建所依赖的工具。
-* 将 ESP-IDF 中的 `tools/` 目录加入 PATH 环境变量中。
-* 运行 `python -m pip install -r requirements.txt` 安装 Python 依赖库。
+请参考入门指南中列出的详细步骤。
 
+* 在主机中安装入门指南中提到的构建所依赖的工具。
+* 运行安装脚本来设置构建环境。可为 Windows shell 选择 `install.bat` 或 `install.ps1`，为 Unix shell 选择 `install.sh` 和 `install.fish`。
+* 在使用 ESP-IDF 之前，需要在 shell 中运行导出脚本。Windows 下可运行 `export.bat`，Unix 下可运行 `source export.sh`。
+  
 ## 配置项目
 
-`idf.py menuconfig`
-
-* 打开项目的文本配置菜单。
-* 使用上下键浏览菜单。
-* 使用回车键进入子菜单，退出键返回上一级菜单或者退出配置。
-* 输入 `?` 查看帮助界面，按下回车键可以退出帮助界面。
-* 使用空格键或者 `Y` 和 `N` 按键来启用和禁用带复选框“`[*]`”的配置项。
-* 高亮某个配置项的同时按下 `?` 键可以显示该选项的帮助文档。
-* 输入 `/` 可以搜索指定的配置项。
-
-一旦配置完成，请按下退出键多次以退出配置界面，当提示是否保存新的的配置时，选择 “Yes”。
+* `idf.py set-target <chip_name>` 可将项目的目标芯片设置为 `<chip_name>`。运行 `idf.py set-target`，不用带任何参数，可查看所有支持的目标芯片列表。
+* `idf.py menuconfig` 可打开一个基于文本的配置菜单，可以用来对项目进行配置。
 
 ## 编译项目
 
@@ -66,23 +70,23 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 
 ## 烧写项目
 
-当构建结束，终端会打印出一条命令行，告知如何使用 esptool.py 工具烧写项目到芯片中。但是你还可以运行下面这条命令来自动烧写：
+当构建结束，终端会打印出一条命令行，告知如何使用 esptool.py 工具烧写项目到芯片中。但你也可以运行下面这条命令来自动烧写：
 
 `idf.py -p PORT flash`
 
-将其中的 PORT 替换为系统中实际串口的名字（比如 Windows 下的 `COM3`，Linux 下的 `/dev/ttyUSB0`，或者 MacOS 下的 `/dev/cu.usbserial-X`。如果省略 `-p` 选项，`idf.py flash` 会尝试使用第一个可用的串口进行烧写。
+将其中的 PORT 替换为系统中实际串口的名字（比如 Windows 下的 `COM3`，Linux 下的 `/dev/ttyUSB0`，或者 macOS 下的 `/dev/cu.usbserial-X`。如果省略 `-p` 选项，`idf.py flash` 会尝试使用第一个可用的串口进行烧写。
 
 这会烧写整个项目（包括应用程序，引导程序和分区表）到芯片中，此外还可以使用 `idf.py menuconfig` 来调整串口烧写相关的配置。
 
-你也不必先运行 `idf.py build`，再运行 `idf.py flash`，`idf.py flash` 会根据需要自动重新构建项目。
+不必先运行 `idf.py build` 再运行 `idf.py flash`，`idf.py flash` 会根据需要自动重新构建项目。
 
 ## 观察串口输入
 
-`idf.py monitor` 会调用 [idf_monitor 工具](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html)来显示 ESP32 和 ESP32-S2 的串口输出。`idf_monitor` 还包含一系列的功能来解析程序崩溃后的输出结果并与设备进行交互。更多详细内容，请参阅[文档](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html).
+`idf.py monitor` 会调用 [idf_monitor 工具](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html)来显示乐鑫芯片的串口输出。`idf_monitor` 还包含一系列的功能来解析程序崩溃后的输出结果并与设备进行交互。更多详细内容，请参阅[文档](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/idf-monitor.html).
 
 输入 `Ctrl-]` 可退出监视器。
 
-想要一次性执行构建，烧写和监视，可以运行如下命令：
+想要一次性执行构建、烧写和监视，可以运行如下命令：
 
 `idf.py flash monitor`
 
@@ -93,15 +97,15 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 * `idf.py app` - 仅构建应用程序。
 * `idf.py app-flash` - 仅烧写应用程序。
 
-`idf.py app-flash` 会自动判断是否有源文件发生了改变而后重新构建应用程序。
+`idf.py app-flash` 会自动判断是否有源文件发生了改变然后重新构建应用程序。
 
 （在正常的开发中，即使引导程序和分区表没有发生变化，每次都重新烧写它们并不会带来什么危害。）
 
 ## 擦除 Flash
 
-`idf.py flash` 并不会擦除 Flash 上所有的内容，但是有时候我们需要设备恢复到完全擦除的状态，尤其是分区表发生了变化或者 OTA 应用升级。要擦除整块 Flash 请运行 `idf.py erase_flash`。
+`idf.py flash` 并不会擦除 flash 上所有的内容，但是有时候我们需要设备恢复到完全擦除的状态，尤其是分区表发生了变化或者 OTA 应用升级时。要擦除整块 flash 请运行 `idf.py erase_flash`。
 
-这条命令还可以和其余命令整合在一起，`idf.py -p PORT erase_flash flash` 会擦除一切然后重新烧写新的应用程序，引导程序和分区表。
+这条命令还可以和其余命令整合在一起，`idf.py -p PORT erase_flash flash` 会擦除一切然后重新烧写新的应用程序、引导程序和分区表。
 
 # 其它参考资源
 
@@ -109,8 +113,6 @@ ESP-IDF 中的子模块采用相对路径（[详见 .gitmodules 文件](.gitmodu
 
 * 可以前往 [esp32.com 论坛](https://esp32.com/) 提问，挖掘社区资源。
 
-* 如果你在使用中发现了错误或者需要新的功能，请先[查看 GitHub Issues](https://github.com/espressif/esp-idf/issues)，确保该问题不会被重复提交。
+* 如果你在使用中发现了错误或者需要新的功能，请先[查看 GitHub Issues](https://github.com/espressif/esp-idf/issues)，确保该问题没有重复提交。
 
 * 如果你有兴趣为 ESP-IDF 作贡献，请先阅读[贡献指南](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/index.html)。
-
-

SUPPORT_POLICY.md

@@ -39,6 +39,10 @@ are not covered by any support period. Sometimes a particular feature is
 marked as "Preview" in a release, which means it is also not covered
 by the support period.
 
+ESP-IDF should be used in an up-to-date software environment. The operating system
+and other third-party tools should be supported by their maintainers.
+ESP-IDF cannot keep compatibility with unsupported third-party tools.
+
 The ESP-IDF Programming Guide has information about the
 [different versions of ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/versions.html)
 (major, minor, bugfix, etc).

components/README.md

@@ -0,0 +1,160 @@
+# Core Components
+
+## Overview
+
+This document contains details about what the core components are, what they contain, and how they are organized.
+
+## Organization
+
+The core components are organized into two groups.
+
+The first group (referred to as `G0` from now on) contains `hal`, `xtensa` and `riscv` (referred to as `arch` components from now on), `esp_rom`, `esp_common`, and `soc`. This
+group contain information about and low-level access to underlying hardware; or in the case of `esp_common`, hardware-agnostic code and utilities.
+These components can depend on each other, but as much as possible have no dependencies outside the group. The reason for this is that, due to the
+nature of what these components contain, the likelihood is high that a lot of other components will require these. Ideally, then, the dependency
+relationship only goes one way. This makes it easier for these components, as a group, to be usable in another project. One can conceivably implement
+a competing SDK to ESP-IDF on top of these components.
+
+The second group (referred to as `G1` from now on) sits at a higher level than the first group. This group contains the components `esp_hw_support`, `esp_system`, `newlib`, `spi_flash`,
+`freertos`, `log`, and `heap`. Like the first group, circular dependencies within the group are allowed; and being at a higher level, dependency on the first group
+is allowed. These components represent software mechanisms essential to building other components.
+
+## Descriptions
+
+The following is a short description of the components mentioned above.
+
+### `G0` Components
+
+#### `hal`
+
+Contains the hardware abstraction layer and low-level operation implementations for the various peripherals. The low-level functions assign meaningful names to register-level manipulations; the hardware abstraction provide operations one level above this, grouping these low-level functions
+into routines that achieve a meaningful action or state of the peripheral.
+
+Example:
+
+- `spi_flash_ll_set_address` is a low-level function part of the hardware abstraction `spi_flash_hal_read_block`
+
+#### `arch`
+
+Contains low-level architecture operations and definitions, including those for customizations (can be thought of on the same level as the low-level functions of `hal`).
+This can also contain files provided by the architecture vendor.
+
+Example:
+
+- `xt_set_exception_handler`
+- `riscv_global_interrupts_enable`
+- `ERI_PERFMON_MAX`
+
+#### `esp_common`
+
+Contains hardware-agnostic definitions, constants, macros, utilities, 'pure' and/or algorithmic functions that is useable by all other components (that is, barring there being a more appropriate component to put them in).
+
+Example:
+
+- `BIT(nr)` and other bit manipulation utilities in the future
+- `IDF_DEPRECATED(REASON)`
+- `ESP_IDF_VERSION_MAJOR`
+
+#### `soc`
+
+Contains description of the underlying hardware: register structure, addresses, pins, capabilities, etc.
+
+Example:
+
+- `DR_REG_DPORT_BASE`
+- `SOC_MCPWM_SUPPORTED`
+- `uart_dev_s`
+
+#### `esp_rom`
+
+Contains headers, linker scripts, abstraction layer, patches, and other related files to ROM functions.
+
+Example:
+
+- `esp32.rom.eco3.ld`
+- `rom/aes.h`
+
+### `G1` Components
+
+#### `spi_flash`
+
+SPI flash device access implementation.
+
+#### `freertos`
+
+FreeRTOS port to targets supported by ESP-IDF.
+
+#### `log`
+
+Logging library.
+
+#### `heap`
+
+Heap implementation.
+
+#### `newlib`
+
+Some functions n the standard library are implemented here, especially those needing other `G1` components.
+
+Example:
+
+- `malloc` is implemented in terms of the component `heap`'s functions
+- `gettimeofday` is implemented in terms of system time in `esp_system`
+
+#### `esp_system`
+
+Contains implementation of system services and controls system behavior. The implementations
+here may take hardware resources and/or decide on a hardware state needed for support of a system service/feature/mechanism.
+Currently, this encompasses the following, but not limited to:
+
+- Startup and initialization
+- Panic and debug
+- Reset and reset reason
+- Task and interrupt watchdogs
+
+#### `esp_hw_support`
+
+Contains implementations that provide hardware operations, arbitration, or resource sharing, especially those that
+is used in the system. Unlike `esp_system`, implementations here do not decide on a hardware state or takes hardware resource, acting
+merely as facilitator to hardware access. Currently, this encompasses the following, but not limited to:
+
+- Interrupt allocation
+- Sleep functions
+- Memory functions (external SPIRAM, async memory, etc.)
+- Clock and clock control
+- Random generation
+- CPU utilities
+- MAC settings
+
+### `esp_hw_support` vs `esp_system`
+
+This section details list some implementations and the reason for placing it in either `esp_hw_support` or `esp_system`.
+
+#### `task_wdt.c` (`esp_system`) vs  `intr_alloc.c` (`esp_hw_support`)
+
+The task watchdog fits the definition of taking and configuring hardware resources (wdt, interrupt) for implementation of a system service/mechanism.
+
+This is in contrast with interrupt allocation that merely facilitates access to the underlying hardware for other implementations -
+drivers, user code, and even the task watchdog mentioned previously!
+
+#### `crosscore_int.c` (`esp_system`)
+
+The current implementation of crosscore interrupts is tightly coupled with a number of interrupt reasons
+associated with system services/mechanisms: REASON_YIELD (scheduler), REASON_FREQ_SWITCH (power management)
+REASON_PRINT_BACKTRACE (panic and debug).
+
+However, if an implementation exists that makes it possible to register an arbitrary interrupt reason - a
+lower level inter-processor call if you will, then this implementation is a good candidate for `esp_hw_support`.
+The current implementation in `esp_system` can then just register the interrupt reasons mentioned above.
+
+#### `esp_mac.h`, `esp_chip_info.h`, `esp_random.h` (`esp_hw_support`)
+
+The functions in these headers used to be in `esp_system.h`, but have been split-off.
+However, to maintain backward compatibility, `esp_system.h` includes these headers.
+
+The remaining functions in `esp_system.h` are those that deal with system behavior, such
+as `esp_register_shutdown_handler`, or are proxy for other system components's APIs such as
+`esp_get_free_heap_size`.
+
+The functions split-off from `esp_system.h` are much more hardware manipulation oriented such as:
+`esp_read_mac`, `esp_random` and `esp_chip_info`.

components/app_trace/CMakeLists.txt

@@ -6,7 +6,23 @@ set(srcs
 
 set(include_dirs "include")
 
-if(CONFIG_SYSVIEW_ENABLE)
+set(priv_include_dirs "private_include" "port/include")
+
+if(CONFIG_APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE)
+    list(APPEND srcs
+        "app_trace_membufs_proto.c")
+
+    if(CONFIG_IDF_TARGET_ARCH_XTENSA)
+        list(APPEND srcs
+            "port/xtensa/port.c")
+    endif()
+    if(CONFIG_IDF_TARGET_ARCH_RISCV)
+        list(APPEND srcs
+            "port/riscv/port.c")
+    endif()
+endif()
+
+if(CONFIG_APPTRACE_SV_ENABLE)
     list(APPEND include_dirs
         sys_view/Config
         sys_view/SEGGER
@@ -16,7 +32,7 @@ if(CONFIG_SYSVIEW_ENABLE)
         "sys_view/SEGGER/SEGGER_SYSVIEW.c"
         "sys_view/Sample/Config/SEGGER_SYSVIEW_Config_FreeRTOS.c"
         "sys_view/Sample/OS/SEGGER_SYSVIEW_FreeRTOS.c"
-        "sys_view/esp32/SEGGER_RTT_esp32.c"
+        "sys_view/esp/SEGGER_RTT_esp.c"
         "sys_view/ext/heap_trace_module.c"
         "sys_view/ext/logging.c")
 endif()
@@ -30,7 +46,8 @@ endif()
 
 idf_component_register(SRCS "${srcs}"
                        INCLUDE_DIRS "${include_dirs}"
-                       PRIV_REQUIRES soc
+                       PRIV_INCLUDE_DIRS "${priv_include_dirs}"
+                       PRIV_REQUIRES soc esp_ipc
                        LDFRAGMENTS linker.lf)
 
 # disable --coverage for this component, as it is used as transport

components/app_trace/Kconfig

@@ -4,29 +4,44 @@ menu "Application Level Tracing"
         prompt "Data Destination"
         default APPTRACE_DEST_NONE
         help
-            Select destination for application trace: trace memory or none (to disable).
+            Select destination for application trace: JTAG or none (to disable).
 
-        config APPTRACE_DEST_TRAX
-            bool "Trace memory"
+        config APPTRACE_DEST_JTAG
+            bool "JTAG"
+            select APPTRACE_DEST_TRAX if IDF_TARGET_ARCH_XTENSA
+            select APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
             select APPTRACE_ENABLE
+
         config APPTRACE_DEST_NONE
             bool "None"
     endchoice
 
-    config APPTRACE_ENABLE
+    config APPTRACE_DEST_TRAX
         bool
-        depends on !ESP32_TRAX && !ESP32S2_TRAX
+        depends on IDF_TARGET_ARCH_XTENSA && !ESP32_TRAX && !ESP32S2_TRAX
         select ESP32_MEMMAP_TRACEMEM
         select ESP32S2_MEMMAP_TRACEMEM
         select ESP32_MEMMAP_TRACEMEM_TWOBANKS
         select ESP32S2_MEMMAP_TRACEMEM_TWOBANKS
         default n
+        help
+            Enables/disable TRAX tracing HW.
+
+    config APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
+        bool
+        default n
+        help
+            Enables/disable swapping memory buffers tracing protocol.
+
+    config APPTRACE_ENABLE
+        bool
+        default n
         help
             Enables/disable application tracing module.
 
     config APPTRACE_LOCK_ENABLE
         bool
-        default !SYSVIEW_ENABLE
+        default !APPTRACE_SV_ENABLE
         help
             Enables/disable application tracing module internal sync lock.
 
@@ -41,16 +56,23 @@ menu "Application Level Tracing"
 
     config APPTRACE_POSTMORTEM_FLUSH_THRESH
         int "Threshold for flushing last trace data to host on panic"
-        depends on APPTRACE_DEST_TRAX
+        depends on APPTRACE_ENABLE
         range 0 16384
         default 0
         help
             Threshold for flushing last trace data to host on panic in post-mortem mode.
             This is minimal amount of data needed to perform flush. In bytes.
 
+    config APPTRACE_BUF_SIZE
+        int "Size of the apptrace buffer"
+        depends on APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE && !APPTRACE_DEST_TRAX
+        default 16384
+        help
+            Size of the memory buffer for trace data in bytes.
+
     config APPTRACE_PENDING_DATA_SIZE_MAX
         int "Size of the pending data buffer"
-        depends on APPTRACE_DEST_TRAX
+        depends on APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
         default 0
         help
             Size of the buffer for events in bytes. It is useful for buffering events from
@@ -59,151 +81,151 @@ menu "Application Level Tracing"
 
     menu "FreeRTOS SystemView Tracing"
         depends on APPTRACE_ENABLE
-        config SYSVIEW_ENABLE
+        config APPTRACE_SV_ENABLE
             bool "SystemView Tracing Enable"
             depends on APPTRACE_ENABLE
             default n
             help
                 Enables supporrt for SEGGER SystemView tracing functionality.
 
-        choice SYSVIEW_TS_SOURCE
+        choice APPTRACE_SV_TS_SOURCE
             prompt "Timer to use as timestamp source"
-            depends on SYSVIEW_ENABLE
-            default SYSVIEW_TS_SOURCE_CCOUNT if FREERTOS_UNICORE && !PM_ENABLE
-            default SYSVIEW_TS_SOURCE_TIMER_00 if !FREERTOS_UNICORE && !PM_ENABLE
-            default SYSVIEW_TS_SOURCE_ESP_TIMER if PM_ENABLE
+            depends on APPTRACE_SV_ENABLE
+            default APPTRACE_SV_TS_SOURCE_CCOUNT if FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_TIMER_00 if !FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
+            default APPTRACE_SV_TS_SOURCE_ESP_TIMER if PM_ENABLE || IDF_TARGET_ESP32C3
             help
                 SystemView needs to use a hardware timer as the source of timestamps
                 when tracing. This option selects the timer for it.
 
-            config SYSVIEW_TS_SOURCE_CCOUNT
+            config APPTRACE_SV_TS_SOURCE_CCOUNT
                 bool "CPU cycle counter (CCOUNT)"
-                depends on FREERTOS_UNICORE && !PM_ENABLE
+                depends on FREERTOS_UNICORE && !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config SYSVIEW_TS_SOURCE_TIMER_00
+            config APPTRACE_SV_TS_SOURCE_TIMER_00
                 bool "Timer 0, Group 0"
-                depends on !PM_ENABLE
+                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config SYSVIEW_TS_SOURCE_TIMER_01
+            config APPTRACE_SV_TS_SOURCE_TIMER_01
                 bool "Timer 1, Group 0"
-                depends on !PM_ENABLE
+                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config SYSVIEW_TS_SOURCE_TIMER_10
+            config APPTRACE_SV_TS_SOURCE_TIMER_10
                 bool "Timer 0, Group 1"
-                depends on !PM_ENABLE
+                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config SYSVIEW_TS_SOURCE_TIMER_11
+            config APPTRACE_SV_TS_SOURCE_TIMER_11
                 bool "Timer 1, Group 1"
-                depends on !PM_ENABLE
+                depends on !PM_ENABLE && !IDF_TARGET_ESP32C3
 
-            config SYSVIEW_TS_SOURCE_ESP_TIMER
+            config APPTRACE_SV_TS_SOURCE_ESP_TIMER
                 bool "esp_timer high resolution timer"
 
         endchoice
 
-        config SYSVIEW_MAX_TASKS
+        config APPTRACE_SV_MAX_TASKS
             int "Maximum supported tasks"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             range 1 64
             default 16
             help
                 Configures maximum supported tasks in sysview debug
 
-        config SYSVIEW_BUF_WAIT_TMO
+        config APPTRACE_SV_BUF_WAIT_TMO
             int "Trace buffer wait timeout"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default 500
             help
                 Configures timeout (in us) to wait for free space in trace buffer.
                 Set to -1 to wait forever and avoid lost events.
 
-        config SYSVIEW_EVT_OVERFLOW_ENABLE
+        config APPTRACE_SV_EVT_OVERFLOW_ENABLE
             bool "Trace Buffer Overflow Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Trace Buffer Overflow" event.
 
-        config SYSVIEW_EVT_ISR_ENTER_ENABLE
+        config APPTRACE_SV_EVT_ISR_ENTER_ENABLE
             bool "ISR Enter Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "ISR Enter" event.
 
-        config SYSVIEW_EVT_ISR_EXIT_ENABLE
+        config APPTRACE_SV_EVT_ISR_EXIT_ENABLE
             bool "ISR Exit Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "ISR Exit" event.
 
-        config SYSVIEW_EVT_ISR_TO_SCHEDULER_ENABLE
+        config APPTRACE_SV_EVT_ISR_TO_SCHED_ENABLE
             bool "ISR Exit to Scheduler Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "ISR to Scheduler" event.
 
-        config SYSVIEW_EVT_TASK_START_EXEC_ENABLE
+        config APPTRACE_SV_EVT_TASK_START_EXEC_ENABLE
             bool "Task Start Execution Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Start Execution" event.
 
-        config SYSVIEW_EVT_TASK_STOP_EXEC_ENABLE
+        config APPTRACE_SV_EVT_TASK_STOP_EXEC_ENABLE
             bool "Task Stop Execution Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Stop Execution" event.
 
-        config SYSVIEW_EVT_TASK_START_READY_ENABLE
+        config APPTRACE_SV_EVT_TASK_START_READY_ENABLE
             bool "Task Start Ready State Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Start Ready State" event.
 
-        config SYSVIEW_EVT_TASK_STOP_READY_ENABLE
+        config APPTRACE_SV_EVT_TASK_STOP_READY_ENABLE
             bool "Task Stop Ready State Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Stop Ready State" event.
 
-        config SYSVIEW_EVT_TASK_CREATE_ENABLE
+        config APPTRACE_SV_EVT_TASK_CREATE_ENABLE
             bool "Task Create Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Create" event.
 
-        config SYSVIEW_EVT_TASK_TERMINATE_ENABLE
+        config APPTRACE_SV_EVT_TASK_TERMINATE_ENABLE
             bool "Task Terminate Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Task Terminate" event.
 
-        config SYSVIEW_EVT_IDLE_ENABLE
+        config APPTRACE_SV_EVT_IDLE_ENABLE
             bool "System Idle Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "System Idle" event.
 
-        config SYSVIEW_EVT_TIMER_ENTER_ENABLE
+        config APPTRACE_SV_EVT_TIMER_ENTER_ENABLE
             bool "Timer Enter Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Timer Enter" event.
 
-        config SYSVIEW_EVT_TIMER_EXIT_ENABLE
+        config APPTRACE_SV_EVT_TIMER_EXIT_ENABLE
             bool "Timer Exit Event"
-            depends on SYSVIEW_ENABLE
+            depends on APPTRACE_SV_ENABLE
             default y
             help
                 Enables "Timer Exit" event.
@@ -212,7 +234,7 @@ menu "Application Level Tracing"
 
     config APPTRACE_GCOV_ENABLE
         bool "GCOV to Host Enable"
-        depends on APPTRACE_ENABLE && !SYSVIEW_ENABLE
+        depends on APPTRACE_ENABLE && !APPTRACE_SV_ENABLE
         select ESP_DEBUG_STUBS_ENABLE
         default n
         help

components/app_trace/app_trace_membufs_proto.c

@@ -0,0 +1,364 @@
+#include <sys/param.h>
+#include <string.h>
+#include "sdkconfig.h"
+#include "esp_log.h"
+#include "esp_app_trace_membufs_proto.h"
+
+/** Trace data header. Every user data chunk is prepended with this header.
+ * User allocates block with esp_apptrace_buffer_get and then fills it with data,
+ * in multithreading environment it can happen that tasks gets buffer and then gets interrupted,
+ * so it is possible that user data are incomplete when  memory block is exposed to the host.
+ * In this case host SW will see that wr_sz < block_sz and will report error.
+ */
+typedef struct {
+#if CONFIG_APPTRACE_SV_ENABLE
+    uint8_t   block_sz; // size of allocated block for user data
+    uint8_t   wr_sz;    // size of actually written data
+#else
+    uint16_t   block_sz; // size of allocated block for user data
+    uint16_t   wr_sz;    // size of actually written data
+#endif
+} esp_tracedata_hdr_t;
+
+/** TODO: docs
+ */
+typedef struct {
+    uint16_t   block_sz; // size of allocated block for user data
+} esp_hostdata_hdr_t;
+
+#if CONFIG_APPTRACE_SV_ENABLE
+#define ESP_APPTRACE_USR_BLOCK_CORE(_cid_)          (0)
+#define ESP_APPTRACE_USR_BLOCK_LEN(_v_)             (_v_)
+#define ESP_APPTRACE_USR_DATA_LEN_MAX(_hw_data_)    255UL
+#else
+#define ESP_APPTRACE_USR_BLOCK_CORE(_cid_)      ((_cid_) << 15)
+#define ESP_APPTRACE_USR_BLOCK_LEN(_v_)         (~(1 << 15) & (_v_))
+#define ESP_APPTRACE_USR_DATA_LEN_MAX(_hw_data_)       (ESP_APPTRACE_INBLOCK(_hw_data_)->sz - sizeof(esp_tracedata_hdr_t))
+#endif
+#define ESP_APPTRACE_USR_BLOCK_RAW_SZ(_s_)     ((_s_) + sizeof(esp_tracedata_hdr_t))
+
+#define ESP_APPTRACE_INBLOCK_MARKER(_hw_data_)          ((_hw_data_)->state.markers[(_hw_data_)->state.in_block % 2])
+#define ESP_APPTRACE_INBLOCK_MARKER_UPD(_hw_data_, _v_)   do {(_hw_data_)->state.markers[(_hw_data_)->state.in_block % 2] += (_v_);}while(0)
+#define ESP_APPTRACE_INBLOCK(_hw_data_)             (&(_hw_data_)->blocks[(_hw_data_)->state.in_block % 2])
+
+const static char *TAG = "esp_apptrace";
+
+static uint32_t esp_apptrace_membufs_down_buffer_write_nolock(esp_apptrace_membufs_proto_data_t *proto, uint8_t *data, uint32_t size);
+
+
+esp_err_t esp_apptrace_membufs_init(esp_apptrace_membufs_proto_data_t *proto, const esp_apptrace_mem_block_t blocks_cfg[2])
+{
+    // disabled by default
+    esp_apptrace_rb_init(&proto->rb_down, NULL, 0);
+    // membufs proto init
+    for (unsigned i = 0; i < 2; i++) {
+        proto->blocks[i].start = blocks_cfg[i].start;
+        proto->blocks[i].sz = blocks_cfg[i].sz;
+        proto->state.markers[i] = 0;
+    }
+    proto->state.in_block = 0;
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    esp_apptrace_rb_init(&proto->rb_pend, proto->pending_data,
+                        sizeof(proto->pending_data));
+#endif
+    return ESP_OK;
+}
+
+void esp_apptrace_membufs_down_buffer_config(esp_apptrace_membufs_proto_data_t *data, uint8_t *buf, uint32_t size)
+{
+    esp_apptrace_rb_init(&data->rb_down, buf, size);
+}
+
+// assumed to be protected by caller from multi-core/thread access
+static esp_err_t esp_apptrace_membufs_swap(esp_apptrace_membufs_proto_data_t *proto)
+{
+    int prev_block_num = proto->state.in_block % 2;
+    int new_block_num = prev_block_num ? (0) : (1);
+    esp_err_t res = ESP_OK;
+
+    res = proto->hw->swap_start(proto->state.in_block);
+    if (res != ESP_OK) {
+        return res;
+    }
+
+    proto->state.markers[new_block_num] = 0;
+    // switch to new block
+    proto->state.in_block++;
+
+    proto->hw->swap(new_block_num);
+
+    // handle data from host
+    esp_hostdata_hdr_t *hdr = (esp_hostdata_hdr_t *)proto->blocks[new_block_num].start;
+    // ESP_APPTRACE_LOGV("Host data %d, sz %d @ %p", proto->hw->host_data_pending(), hdr->block_sz, hdr);
+    if (proto->hw->host_data_pending() && hdr->block_sz > 0) {
+        // TODO: add support for multiple blocks from host, currently there is no need for that
+        uint8_t *p = proto->blocks[new_block_num].start + proto->blocks[new_block_num].sz;
+        ESP_APPTRACE_LOGD("Recvd %d bytes from host [%x %x %x %x %x %x %x %x .. %x %x %x %x %x %x %x %x]", hdr->block_sz,
+            *(proto->blocks[new_block_num].start+0), *(proto->blocks[new_block_num].start+1),
+            *(proto->blocks[new_block_num].start+2), *(proto->blocks[new_block_num].start+3),
+            *(proto->blocks[new_block_num].start+4), *(proto->blocks[new_block_num].start+5),
+            *(proto->blocks[new_block_num].start+6), *(proto->blocks[new_block_num].start+7),
+            *(p-8), *(p-7), *(p-6), *(p-5), *(p-4), *(p-3), *(p-2), *(p-1));
+        uint32_t sz = esp_apptrace_membufs_down_buffer_write_nolock(proto, (uint8_t *)(hdr+1), hdr->block_sz);
+        if (sz != hdr->block_sz) {
+            ESP_APPTRACE_LOGE("Failed to write %d bytes to down buffer (%d %d)!", hdr->block_sz - sz, hdr->block_sz, sz);
+        }
+        hdr->block_sz = 0;
+    }
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    // copy pending data to  block if any
+    while (proto->state.markers[new_block_num] < proto->blocks[new_block_num].sz) {
+        uint32_t read_sz = esp_apptrace_rb_read_size_get(&proto->rb_pend);
+        if (read_sz == 0) {
+            break; // no more data in pending buffer
+        }
+        if (read_sz > proto->blocks[new_block_num].sz - proto->state.markers[new_block_num]) {
+            read_sz = proto->blocks[new_block_num].sz - proto->state.markers[new_block_num];
+        }
+        uint8_t *ptr = esp_apptrace_rb_consume(&proto->rb_pend, read_sz);
+        if (!ptr) {
+            assert(false && "Failed to consume pended bytes!!");
+            break;
+        }
+        ESP_APPTRACE_LOGD("Pump %d pend bytes [%x %x %x %x : %x %x %x %x : %x %x %x %x : %x %x...%x %x]",
+            read_sz, *(ptr+0), *(ptr+1), *(ptr+2), *(ptr+3), *(ptr+4),
+            *(ptr+5), *(ptr+6), *(ptr+7), *(ptr+8), *(ptr+9), *(ptr+10), *(ptr+11), *(ptr+12), *(ptr+13), *(ptr+read_sz-2), *(ptr+read_sz-1));
+        memcpy(proto->blocks[new_block_num].start + proto->state.markers[new_block_num], ptr, read_sz);
+        proto->state.markers[new_block_num] += read_sz;
+    }
+#endif
+    proto->hw->swap_end(proto->state.in_block, proto->state.markers[prev_block_num]);
+    return res;
+}
+
+static esp_err_t esp_apptrace_membufs_swap_waitus(esp_apptrace_membufs_proto_data_t *proto, esp_apptrace_tmo_t *tmo)
+{
+    int res;
+
+    while ((res = esp_apptrace_membufs_swap(proto)) != ESP_OK) {
+        res = esp_apptrace_tmo_check(tmo);
+        if (res != ESP_OK) {
+            break;
+        }
+    }
+    return res;
+}
+
+uint8_t *esp_apptrace_membufs_down_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr = NULL;
+
+    while (1) {
+        uint32_t sz = esp_apptrace_rb_read_size_get(&proto->rb_down);
+        if (sz != 0) {
+            *size = MIN(*size, sz);
+            ptr = esp_apptrace_rb_consume(&proto->rb_down, *size);
+            if (!ptr) {
+                assert(false && "Failed to consume bytes from down buffer!");
+            }
+            break;
+        }
+        // may need to flush
+        if (proto->hw->host_data_pending()) {
+            ESP_APPTRACE_LOGD("force flush");
+            int res = esp_apptrace_membufs_swap_waitus(proto, tmo);
+            if (res != ESP_OK) {
+                ESP_APPTRACE_LOGE("Failed to switch to another block to recv data from host!");
+                /*do not return error because data can be in down buffer already*/
+            }
+        } else {
+            // check tmo only if there is no data from host
+            int res = esp_apptrace_tmo_check(tmo);
+            if (res != ESP_OK) {
+                return NULL;
+            }
+        }
+    }
+    return ptr;
+}
+
+esp_err_t esp_apptrace_membufs_down_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    /* nothing todo */
+    return ESP_OK;
+}
+
+static uint32_t esp_apptrace_membufs_down_buffer_write_nolock(esp_apptrace_membufs_proto_data_t *proto, uint8_t *data, uint32_t size)
+{
+    uint32_t total_sz = 0;
+
+    while (total_sz < size) {
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock WRS %d-%d-%d %d", proto->rb_down.wr, proto->rb_down.rd,
+            proto->rb_down.cur_size, size);
+        uint32_t wr_sz = esp_apptrace_rb_write_size_get(&proto->rb_down);
+        if (wr_sz == 0) {
+            break;
+        }
+
+        if (wr_sz > size - total_sz) {
+            wr_sz = size - total_sz;
+        }
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d", wr_sz);
+        uint8_t *ptr = esp_apptrace_rb_produce(&proto->rb_down, wr_sz);
+        if (!ptr) {
+            assert(false && "Failed to produce bytes to down buffer!");
+        }
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d to 0x%x from 0x%x", wr_sz, ptr, data + total_sz + wr_sz);
+        memcpy(ptr, data + total_sz, wr_sz);
+        total_sz += wr_sz;
+        ESP_APPTRACE_LOGD("esp_apptrace_trax_down_buffer_write_nolock wr %d/%d", wr_sz, total_sz);
+    }
+    return total_sz;
+}
+
+static inline uint8_t *esp_apptrace_membufs_wait4buf(esp_apptrace_membufs_proto_data_t *proto, uint16_t size, esp_apptrace_tmo_t *tmo, int *pended)
+{
+    uint8_t *ptr = NULL;
+
+    int res = esp_apptrace_membufs_swap_waitus(proto, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    // check if we still have pending data
+    if (esp_apptrace_rb_read_size_get(&proto->rb_pend) > 0) {
+        // if after block switch we still have pending data (not all pending data have been pumped to block)
+        // alloc new pending buffer
+        *pended = 1;
+        ptr = esp_apptrace_rb_produce(&proto->rb_pend, size);
+        if (!ptr) {
+            ESP_APPTRACE_LOGE("Failed to alloc pend buf 1: w-r-s %d-%d-%d!", proto->rb_pend.wr, proto->rb_pend.rd, proto->rb_pend.cur_size);
+        }
+    } else
+#endif
+    {
+        // update block pointers
+        if (ESP_APPTRACE_INBLOCK_MARKER(proto) + size > ESP_APPTRACE_INBLOCK(proto)->sz) {
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+            *pended = 1;
+            ptr = esp_apptrace_rb_produce(&proto->rb_pend, size);
+            if (ptr == NULL) {
+                ESP_APPTRACE_LOGE("Failed to alloc pend buf 2: w-r-s %d-%d-%d!", proto->rb_pend.wr, proto->rb_pend.rd, proto->rb_pend.cur_size);
+            }
+#endif
+        } else {
+            *pended = 0;
+            ptr = ESP_APPTRACE_INBLOCK(proto)->start + ESP_APPTRACE_INBLOCK_MARKER(proto);
+        }
+    }
+
+    return ptr;
+}
+
+static inline uint8_t *esp_apptrace_membufs_pkt_start(uint8_t *ptr, uint16_t size)
+{
+    // it is safe to use cpu_hal_get_core_id() in macro call because arg is used only once inside it
+    ((esp_tracedata_hdr_t *)ptr)->block_sz = ESP_APPTRACE_USR_BLOCK_CORE(cpu_hal_get_core_id()) | size;
+    ((esp_tracedata_hdr_t *)ptr)->wr_sz = 0;
+    return ptr + sizeof(esp_tracedata_hdr_t);
+}
+
+static inline void esp_apptrace_membufs_pkt_end(uint8_t *ptr)
+{
+    esp_tracedata_hdr_t *hdr = (esp_tracedata_hdr_t *)(ptr - sizeof(esp_tracedata_hdr_t));
+    // update written size
+    hdr->wr_sz = hdr->block_sz;
+}
+
+uint8_t *esp_apptrace_membufs_up_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *buf_ptr = NULL;
+
+    if (size > ESP_APPTRACE_USR_DATA_LEN_MAX(proto)) {
+        ESP_APPTRACE_LOGE("Too large user data size %d!", size);
+        return NULL;
+    }
+
+    // check for data in the pending buffer
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    if (esp_apptrace_rb_read_size_get(&proto->rb_pend) > 0) {
+        // if we have buffered data try to switch  block
+        esp_apptrace_membufs_swap(proto);
+        // if switch was successful, part or all pended data have been copied to  block
+    }
+    if (esp_apptrace_rb_read_size_get(&proto->rb_pend) > 0) {
+        // if we have buffered data alloc new pending buffer
+        ESP_APPTRACE_LOGD("Get %d bytes from PEND buffer", size);
+        buf_ptr = esp_apptrace_rb_produce(&proto->rb_pend, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+        if (buf_ptr == NULL) {
+            int pended_buf;
+            buf_ptr = esp_apptrace_membufs_wait4buf(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size), tmo, &pended_buf);
+            if (buf_ptr && !pended_buf) {
+                ESP_APPTRACE_LOGD("Get %d bytes from block", size);
+                // update cur block marker
+                ESP_APPTRACE_INBLOCK_MARKER_UPD(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+            }
+        }
+    } else {
+#else
+    if (1) {
+#endif
+        if (ESP_APPTRACE_INBLOCK_MARKER(proto) + ESP_APPTRACE_USR_BLOCK_RAW_SZ(size) > ESP_APPTRACE_INBLOCK(proto)->sz) {
+            #if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+            ESP_APPTRACE_LOGD("Block full. Get %d bytes from PEND buffer", size);
+            buf_ptr = esp_apptrace_rb_produce(&proto->rb_pend, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+            #endif
+            if (buf_ptr == NULL) {
+                int pended_buf;
+                ESP_APPTRACE_LOGD(" full. Get %d bytes from pend buffer", size);
+                buf_ptr = esp_apptrace_membufs_wait4buf(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size), tmo, &pended_buf);
+                if (buf_ptr && !pended_buf) {
+                    ESP_APPTRACE_LOGD("Got %d bytes from block", size);
+                    // update cur block marker
+                    ESP_APPTRACE_INBLOCK_MARKER_UPD(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+                }
+            }
+        } else {
+            ESP_APPTRACE_LOGD("Get %d bytes from  buffer", size);
+            // fit to curr  nlock
+            buf_ptr = ESP_APPTRACE_INBLOCK(proto)->start + ESP_APPTRACE_INBLOCK_MARKER(proto);
+            // update cur block marker
+            ESP_APPTRACE_INBLOCK_MARKER_UPD(proto, ESP_APPTRACE_USR_BLOCK_RAW_SZ(size));
+        }
+    }
+    if (buf_ptr) {
+        buf_ptr = esp_apptrace_membufs_pkt_start(buf_ptr, size);
+    }
+
+    return buf_ptr;
+}
+
+esp_err_t esp_apptrace_membufs_up_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    esp_apptrace_membufs_pkt_end(ptr);
+    // TODO: mark block as busy in order not to re-use it for other tracing calls until it is completely written
+    // TODO: avoid potential situation when all memory is consumed by low prio tasks which can not complete writing due to
+    // higher prio tasks and the latter can not allocate buffers at all
+    // this is abnormal situation can be detected on host which will receive only uncompleted buffers
+    // workaround: use own memcpy which will kick-off dead tracing calls
+    return ESP_OK;
+}
+
+esp_err_t esp_apptrace_membufs_flush_nolock(esp_apptrace_membufs_proto_data_t *proto, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    int res = ESP_OK;
+
+    if (ESP_APPTRACE_INBLOCK_MARKER(proto) < min_sz) {
+        ESP_APPTRACE_LOGI("Ignore flush request for min %d bytes. Bytes in  block: %d.", min_sz, ESP_APPTRACE_INBLOCK_MARKER(proto));
+        return ESP_OK;
+    }
+    // switch  block while size of data (including that in pending buffer) is more than min size
+    while (ESP_APPTRACE_INBLOCK_MARKER(proto) > min_sz) {
+        ESP_APPTRACE_LOGD("Try to flush %d bytes. Wait until block switch for %lld us", ESP_APPTRACE_INBLOCK_MARKER(proto), tmo->tmo);
+        res = esp_apptrace_membufs_swap_waitus(proto, tmo);
+        if (res != ESP_OK) {
+            if (tmo->tmo != ESP_APPTRACE_TMO_INFINITE)
+                ESP_APPTRACE_LOGW("Failed to switch to another block in %lld us!", tmo->tmo);
+            else
+                ESP_APPTRACE_LOGE("Failed to switch to another block in %lld us!", tmo->tmo);
+            return res;
+        }
+    }
+
+    return res;
+}

components/app_trace/app_trace_util.c

@@ -8,33 +8,42 @@
 #include "freertos/task.h"
 #include "esp_app_trace_util.h"
 #include "sdkconfig.h"
-#if CONFIG_IDF_TARGET_ESP32
-#include "esp32/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S2
-#include "esp32s2/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32S3
-#include "esp32s3/clk.h"
-#elif CONFIG_IDF_TARGET_ESP32C3
-#include "esp32c3/clk.h"
+
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////// Locks /////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+
+#if ESP_APPTRACE_PRINT_LOCK
+static esp_apptrace_lock_t s_log_lock = {.irq_stat = 0, .portmux = portMUX_INITIALIZER_UNLOCKED};
+#endif
+
+int esp_apptrace_log_lock(void)
+{
+#if ESP_APPTRACE_PRINT_LOCK
+    esp_apptrace_tmo_t tmo;
+    esp_apptrace_tmo_init(&tmo, ESP_APPTRACE_TMO_INFINITE);
+    int ret = esp_apptrace_lock_take(&s_log_lock, &tmo);
+    return ret;
+#else
+    return 0;
 #endif
+}
+
+void esp_apptrace_log_unlock(void)
+{
+ #if ESP_APPTRACE_PRINT_LOCK
+    esp_apptrace_lock_give(&s_log_lock);
+#endif
+}
 
 ///////////////////////////////////////////////////////////////////////////////
 ///////////////////////////////// TIMEOUT /////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
 
-#define ESP_APPTRACE_CPUTICKS2US(_t_, _cpu_freq_)       ((_t_)/(_cpu_freq_/1000000))
-#define ESP_APPTRACE_US2CPUTICKS(_t_, _cpu_freq_)       ((_t_)*(_cpu_freq_/1000000))
-
 esp_err_t esp_apptrace_tmo_check(esp_apptrace_tmo_t *tmo)
 {
-    int cpu_freq = esp_clk_cpu_freq();
-    if (tmo->tmo != ESP_APPTRACE_TMO_INFINITE) {
-        unsigned cur = portGET_RUN_TIME_COUNTER_VALUE();
-        if (tmo->start <= cur) {
-            tmo->elapsed = ESP_APPTRACE_CPUTICKS2US(cur - tmo->start, cpu_freq);
-        } else {
-            tmo->elapsed = ESP_APPTRACE_CPUTICKS2US(0xFFFFFFFF - tmo->start + cur, cpu_freq);
-        }
+    if (tmo->tmo != (int64_t)-1) {
+        tmo->elapsed = esp_timer_get_time() - tmo->start;
         if (tmo->elapsed >= tmo->tmo) {
             return ESP_ERR_TIMEOUT;
         }

components/app_trace/component.mk

@@ -4,14 +4,21 @@
 
 COMPONENT_SRCDIRS := .
 
+ifdef CONFIG_APPTRACE_MEMBUFS_APPTRACE_PROTO_ENABLE
+COMPONENT_SRCDIRS += port/xtensa
+endif
+
 COMPONENT_ADD_INCLUDEDIRS = include
 
+COMPONENT_PRIV_INCLUDEDIRS = private_include \
+	port/include
+
 COMPONENT_ADD_LDFLAGS = -lapp_trace
 
 # do not produce gcov info for this module, it is used as transport for gcov
 CFLAGS := $(subst --coverage,,$(CFLAGS))
 
-ifdef CONFIG_SYSVIEW_ENABLE
+ifdef CONFIG_APPTRACE_SV_ENABLE
 
 COMPONENT_ADD_INCLUDEDIRS += \
 	sys_view/Config \
@@ -23,7 +30,7 @@ COMPONENT_SRCDIRS += \
 	sys_view/SEGGER \
 	sys_view/Sample/OS \
 	sys_view/Sample/Config \
-	sys_view/esp32 \
+	sys_view/esp \
 	sys_view/ext
 else
 COMPONENT_SRCDIRS += gcov

components/app_trace/gcov/gcov_rtio.c

@@ -14,7 +14,9 @@
 #include "soc/cpu.h"
 #include "soc/timer_periph.h"
 #include "esp_app_trace.h"
+#include "esp_freertos_hooks.h"
 #include "esp_private/dbg_stubs.h"
+#include "esp_ipc.h"
 #include "hal/wdt_hal.h"
 #if CONFIG_IDF_TARGET_ESP32
 #include "esp32/rom/libc_stubs.h"
@@ -28,128 +30,113 @@
 
 #include "esp_log.h"
 const static char *TAG = "esp_gcov_rtio";
+static volatile bool s_create_gcov_task = false;
+static volatile bool s_gcov_task_running = false;
 
 extern void __gcov_dump(void);
 extern void __gcov_reset(void);
 
-static struct syscall_stub_table s_gcov_stub_table;
-
-
-static int gcov_stub_lock_try_acquire_recursive(_lock_t *lock)
+void gcov_dump_task(void *pvParameter)
 {
-    if (*lock && uxSemaphoreGetCount((xSemaphoreHandle)(*lock)) == 0) {
-        // we can do nothing here, gcov dump is initiated with some resource locked
-        // which is also used by gcov functions
-        ESP_EARLY_LOGE(TAG, "Lock 0x%x is busy during GCOV dump! System state can be inconsistent after dump!", lock);
-    }
-    return pdTRUE;
-}
+    int dump_result = 0;
+    bool *running = (bool *)pvParameter;
 
-static void gcov_stub_lock_acquire_recursive(_lock_t *lock)
-{
-    gcov_stub_lock_try_acquire_recursive(lock);
-}
-
-static void gcov_stub_lock_release_recursive(_lock_t *lock)
-{
-}
+    ESP_EARLY_LOGV(TAG, "%s stack use in %d", __FUNCTION__, uxTaskGetStackHighWaterMark(NULL));
 
-static int esp_dbg_stub_gcov_dump_do(void)
-{
-    int ret = ESP_OK;
-    FILE* old_stderr = stderr;
-    FILE* old_stdout = stdout;
-    static struct syscall_stub_table *old_tables[portNUM_PROCESSORS];
-
-    old_tables[0] = syscall_table_ptr_pro;
-#if portNUM_PROCESSORS > 1
-    old_tables[1] = syscall_table_ptr_app;
-#endif
     ESP_EARLY_LOGV(TAG, "Alloc apptrace down buf %d bytes", ESP_GCOV_DOWN_BUF_SIZE);
     void *down_buf = malloc(ESP_GCOV_DOWN_BUF_SIZE);
     if (down_buf == NULL) {
         ESP_EARLY_LOGE(TAG, "Could not allocate memory for the buffer");
-        return ESP_ERR_NO_MEM;
+        dump_result = ESP_ERR_NO_MEM;
+        goto gcov_exit;
     }
     ESP_EARLY_LOGV(TAG, "Config apptrace down buf");
     esp_apptrace_down_buffer_config(down_buf, ESP_GCOV_DOWN_BUF_SIZE);
+    /* we are directing the std outputs to the fake ones in order to reduce stack usage */
+    FILE *old_stderr = stderr;
+    FILE *old_stdout = stdout;
+    stderr = (FILE *) &__sf_fake_stderr;
+    stdout = (FILE *) &__sf_fake_stdout;
     ESP_EARLY_LOGV(TAG, "Dump data...");
-    // incase of dual-core chip APP and PRO CPUs share the same table, so it is safe to save only PRO's table
-    memcpy(&s_gcov_stub_table, syscall_table_ptr_pro, sizeof(s_gcov_stub_table));
-    s_gcov_stub_table._lock_acquire_recursive = &gcov_stub_lock_acquire_recursive;
-    s_gcov_stub_table._lock_release_recursive = &gcov_stub_lock_release_recursive;
-    s_gcov_stub_table._lock_try_acquire_recursive = &gcov_stub_lock_try_acquire_recursive,
-    syscall_table_ptr_pro = &s_gcov_stub_table;
-#if portNUM_PROCESSORS > 1
-    syscall_table_ptr_app = &s_gcov_stub_table;
-#endif
-    stderr = (FILE*) &__sf_fake_stderr;
-    stdout = (FILE*) &__sf_fake_stdout;
     __gcov_dump();
     // reset dump status to allow incremental data accumulation
     __gcov_reset();
-    stdout = old_stdout;
-    stderr = old_stderr;
-    syscall_table_ptr_pro = old_tables[0];
-#if portNUM_PROCESSORS > 1
-    syscall_table_ptr_app = old_tables[1];
-#endif
-    ESP_EARLY_LOGV(TAG, "Free apptrace down buf");
     free(down_buf);
+    stderr = old_stderr;
+    stdout = old_stdout;
     ESP_EARLY_LOGV(TAG, "Finish file transfer session");
-    ret = esp_apptrace_fstop(ESP_APPTRACE_DEST_TRAX);
-    if (ret != ESP_OK) {
-        ESP_EARLY_LOGE(TAG, "Failed to send files transfer stop cmd (%d)!", ret);
+    dump_result = esp_apptrace_fstop(ESP_APPTRACE_DEST_TRAX);
+    if (dump_result != ESP_OK) {
+        ESP_EARLY_LOGE(TAG, "Failed to send files transfer stop cmd (%d)!", dump_result);
+    }
+
+gcov_exit:
+    ESP_EARLY_LOGV(TAG, "dump_result %d", dump_result);
+    if (running) {
+        *running = false;
+    }
+
+    ESP_EARLY_LOGV(TAG, "%s stack use out %d", __FUNCTION__, uxTaskGetStackHighWaterMark(NULL));
+
+    vTaskDelete(NULL);
+}
+
+void gcov_create_task(void *arg)
+{
+    ESP_EARLY_LOGV(TAG, "%s", __FUNCTION__);
+    xTaskCreatePinnedToCore(&gcov_dump_task, "gcov_dump_task", 2048, (void *)&s_gcov_task_running, configMAX_PRIORITIES - 1, NULL, 0);
+}
+
+void gcov_create_task_tick_hook(void)
+{
+    extern esp_err_t esp_ipc_start_gcov_from_isr(uint32_t cpu_id, esp_ipc_func_t func, void* arg);
+    if (s_create_gcov_task) {
+        if (esp_ipc_start_gcov_from_isr(xPortGetCoreID(), &gcov_create_task, NULL) == ESP_OK) {
+            s_create_gcov_task = false;
+        }
     }
-    ESP_EARLY_LOGV(TAG, "exit %d", ret);
-    return ret;
 }
 
 /**
- * @brief Triggers gcov info dump.
+ * @brief Triggers gcov info dump task
  *        This function is to be called by OpenOCD, not by normal user code.
- * TODO: what about interrupted flash access (when cache disabled)???
+ * TODO: what about interrupted flash access (when cache disabled)
  *
  * @return ESP_OK on success, otherwise see esp_err_t
  */
 static int esp_dbg_stub_gcov_entry(void)
 {
-    return esp_dbg_stub_gcov_dump_do();
+    /* we are in isr context here */
+    s_create_gcov_task = true;
+    return ESP_OK;
 }
 
 int gcov_rtio_atexit(void (*function)(void) __attribute__ ((unused)))
 {
+    uint32_t capabilities = 0;
     ESP_EARLY_LOGV(TAG, "%s", __FUNCTION__);
     esp_dbg_stub_entry_set(ESP_DBG_STUB_ENTRY_GCOV, (uint32_t)&esp_dbg_stub_gcov_entry);
-    return 0;
+    if (esp_dbg_stub_entry_get(ESP_DBG_STUB_ENTRY_CAPABILITIES, &capabilities) == ESP_OK) {
+        esp_dbg_stub_entry_set(ESP_DBG_STUB_ENTRY_CAPABILITIES, capabilities | ESP_DBG_STUB_CAP_GCOV_TASK);
+    }
+    esp_register_freertos_tick_hook(gcov_create_task_tick_hook);
+    return ESP_OK;
 }
 
 void esp_gcov_dump(void)
 {
-    // disable IRQs on this CPU, other CPU is halted by OpenOCD
-    unsigned irq_state = portENTER_CRITICAL_NESTED();
-#if !CONFIG_FREERTOS_UNICORE
-    int other_core = xPortGetCoreID() ? 0 : 1;
-    esp_cpu_stall(other_core);
-#endif
+    ESP_EARLY_LOGV(TAG, "%s", __FUNCTION__);
+
     while (!esp_apptrace_host_is_connected(ESP_APPTRACE_DEST_TRAX)) {
-        wdt_hal_context_t twdt = {.inst = WDT_MWDT0, .mwdt_dev = &TIMERG0};
-        wdt_hal_context_t iwdt = {.inst = WDT_MWDT1, .mwdt_dev = &TIMERG1};
-        //Feed the Task Watchdog (TG0) to prevent it from timing out
-        wdt_hal_write_protect_disable(&twdt);
-        wdt_hal_feed(&twdt);
-        wdt_hal_write_protect_enable(&twdt);
-        //Likewise, feed the Interrupt Watchdog (TG1) to prevent a reboot
-        wdt_hal_write_protect_disable(&iwdt);
-        wdt_hal_feed(&iwdt);
-        wdt_hal_write_protect_enable(&iwdt);
+        vTaskDelay(pdMS_TO_TICKS(10));
     }
 
-    esp_dbg_stub_gcov_dump_do();
-#if !CONFIG_FREERTOS_UNICORE
-    esp_cpu_unstall(other_core);
-#endif
-    portEXIT_CRITICAL_NESTED(irq_state);
+    /* We are not in isr context here. Waiting for the completion is safe */
+    s_gcov_task_running = true;
+    s_create_gcov_task = true;
+    while (s_gcov_task_running) {
+        vTaskDelay(pdMS_TO_TICKS(10));
+    }
 }
 
 void *gcov_rtio_fopen(const char *path, const char *mode)
@@ -168,7 +155,7 @@ int gcov_rtio_fclose(void *stream)
 
 size_t gcov_rtio_fread(void *ptr, size_t size, size_t nmemb, void *stream)
 {
-    ESP_EARLY_LOGV(TAG, "%s read %u", __FUNCTION__, size*nmemb);
+    ESP_EARLY_LOGV(TAG, "%s read %u", __FUNCTION__, size * nmemb);
     size_t sz = esp_apptrace_fread(ESP_APPTRACE_DEST_TRAX, ptr, size, nmemb, stream);
     ESP_EARLY_LOGV(TAG, "%s actually read %u", __FUNCTION__, sz);
     return sz;

components/app_trace/heap_trace_tohost.c

@@ -9,7 +9,7 @@
 #include "esp_heap_trace.h"
 #undef HEAP_TRACE_SRCFILE
 
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
 #include "esp_app_trace.h"
 #include "esp_sysview_trace.h"
 #endif
@@ -18,7 +18,7 @@
 
 #ifdef CONFIG_HEAP_TRACING_TOHOST
 
-#if !CONFIG_SYSVIEW_ENABLE
+#if !CONFIG_APPTRACE_SV_ENABLE
 #error None of the heap tracing backends is enabled! You must enable SystemView compatible tracing to use this feature.
 #endif
 
@@ -34,7 +34,7 @@ esp_err_t heap_trace_init_tohost(void)
 
 esp_err_t heap_trace_start(heap_trace_mode_t mode_param)
 {
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     esp_err_t ret = esp_sysview_heap_trace_start((uint32_t)-1);
     if (ret != ESP_OK) {
         return ret;
@@ -47,7 +47,7 @@ esp_err_t heap_trace_start(heap_trace_mode_t mode_param)
 esp_err_t heap_trace_stop(void)
 {
     esp_err_t ret = ESP_ERR_NOT_SUPPORTED;
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     ret = esp_sysview_heap_trace_stop();
 #endif
     s_tracing = false;
@@ -80,7 +80,7 @@ static IRAM_ATTR void record_allocation(const heap_trace_record_t *record)
     if (!s_tracing) {
         return;
     }
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     esp_sysview_heap_trace_alloc(record->address, record->size, record->alloced_by);
 #endif
 }
@@ -95,7 +95,7 @@ static IRAM_ATTR void record_free(void *p, void **callers)
     if (!s_tracing) {
         return;
     }
-#if CONFIG_SYSVIEW_ENABLE
+#if CONFIG_APPTRACE_SV_ENABLE
     esp_sysview_heap_trace_free(p, callers);
 #endif
 }

components/app_trace/include/esp_app_trace.h

@@ -18,8 +18,11 @@ extern "C" {
  * Application trace data destinations bits.
  */
 typedef enum {
-    ESP_APPTRACE_DEST_TRAX = 0x1,	///< JTAG destination
-    ESP_APPTRACE_DEST_UART0 = 0x2,	///< UART destination
+    ESP_APPTRACE_DEST_JTAG = 1,                         ///< JTAG destination
+    ESP_APPTRACE_DEST_TRAX = ESP_APPTRACE_DEST_JTAG, 	///< xxx_TRAX name is obsolete, use more common xxx_JTAG
+    ESP_APPTRACE_DEST_UART0,	                        ///< UART0 destination
+    ESP_APPTRACE_DEST_MAX = ESP_APPTRACE_DEST_UART0,
+    ESP_APPTRACE_DEST_NUM
 } esp_apptrace_dest_t;
 
 /**

components/app_trace/include/esp_app_trace_util.h

@@ -12,6 +12,7 @@ extern "C" {
 
 #include "freertos/FreeRTOS.h"
 #include "esp_err.h"
+#include "esp_timer.h"
 
 /** Infinite waiting timeout */
 #define ESP_APPTRACE_TMO_INFINITE               ((uint32_t)-1)
@@ -22,9 +23,9 @@ extern "C" {
  *  periodically to check timeout for expiration.
  */
 typedef struct {
-    uint32_t   start;   ///< time interval start (in CPU ticks)
-    uint32_t   tmo;     ///< timeout value (in us)
-    uint32_t   elapsed; ///< elapsed time (in us)
+    int64_t   start;   ///< time interval start (in us)
+    int64_t   tmo;     ///< timeout value (in us)
+    int64_t   elapsed; ///< elapsed time (in us)
 } esp_apptrace_tmo_t;
 
 /**
@@ -35,23 +36,23 @@ typedef struct {
 */
 static inline void esp_apptrace_tmo_init(esp_apptrace_tmo_t *tmo, uint32_t user_tmo)
 {
-    tmo->start = portGET_RUN_TIME_COUNTER_VALUE();
-    tmo->tmo = user_tmo;
+    tmo->start = esp_timer_get_time();
+    tmo->tmo = user_tmo == ESP_APPTRACE_TMO_INFINITE ? (int64_t)-1 : (int64_t)user_tmo;
     tmo->elapsed = 0;
 }
 
 /**
  * @brief Checks timeout for expiration.
  *
- * @param tmo Pointer to timeout structure to be initialized.
+ * @param tmo Pointer to timeout structure.
  *
- * @return ESP_OK on success, otherwise \see esp_err_t
+ * @return number of remaining us till tmo.
  */
 esp_err_t esp_apptrace_tmo_check(esp_apptrace_tmo_t *tmo);
 
 static inline uint32_t esp_apptrace_tmo_remaining_us(esp_apptrace_tmo_t *tmo)
 {
-    return tmo->tmo != ESP_APPTRACE_TMO_INFINITE ? (tmo->elapsed - tmo->tmo) : ESP_APPTRACE_TMO_INFINITE;
+    return tmo->tmo != (int64_t)-1 ? (tmo->elapsed - tmo->tmo) : ESP_APPTRACE_TMO_INFINITE;
 }
 
 /** Tracing module synchronization lock */
@@ -160,6 +161,30 @@ uint32_t esp_apptrace_rb_read_size_get(esp_apptrace_rb_t *rb);
  */
 uint32_t esp_apptrace_rb_write_size_get(esp_apptrace_rb_t *rb);
 
+int esp_apptrace_log_lock(void);
+void esp_apptrace_log_unlock(void);
+
+#define ESP_APPTRACE_LOG( format, ... )   \
+    do { \
+        esp_apptrace_log_lock(); \
+        esp_rom_printf(format, ##__VA_ARGS__); \
+        esp_apptrace_log_unlock(); \
+    } while(0)
+
+#define ESP_APPTRACE_LOG_LEV( _L_, level, format, ... )   \
+    do { \
+        if (LOG_LOCAL_LEVEL >= level) { \
+            ESP_APPTRACE_LOG(LOG_FORMAT(_L_, format), esp_log_early_timestamp(), TAG, ##__VA_ARGS__); \
+        } \
+    } while(0)
+
+#define ESP_APPTRACE_LOGE( format, ... )  ESP_APPTRACE_LOG_LEV(E, ESP_LOG_ERROR, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGW( format, ... )  ESP_APPTRACE_LOG_LEV(W, ESP_LOG_WARN, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGI( format, ... )  ESP_APPTRACE_LOG_LEV(I, ESP_LOG_INFO, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGD( format, ... )  ESP_APPTRACE_LOG_LEV(D, ESP_LOG_DEBUG, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGV( format, ... )  ESP_APPTRACE_LOG_LEV(V, ESP_LOG_VERBOSE, format, ##__VA_ARGS__)
+#define ESP_APPTRACE_LOGO( format, ... )  ESP_APPTRACE_LOG_LEV(E, ESP_LOG_NONE, format, ##__VA_ARGS__)
+
 #ifdef __cplusplus
 }
 #endif

components/app_trace/include/esp_sysview_trace.h

@@ -12,7 +12,7 @@ extern "C" {
 
 #include <stdarg.h>
 #include "esp_err.h"
-#include "SEGGER_RTT.h" // SEGGER_RTT_ESP32_Flush
+#include "SEGGER_RTT.h" // SEGGER_RTT_ESP_Flush
 #include "esp_app_trace_util.h" // ESP_APPTRACE_TMO_INFINITE
 
 /**
@@ -24,7 +24,7 @@ extern "C" {
  */
 static inline esp_err_t esp_sysview_flush(uint32_t tmo)
 {
-    SEGGER_RTT_ESP32_Flush(0, tmo);
+    SEGGER_RTT_ESP_Flush(0, tmo);
     return ESP_OK;
 }
 

components/app_trace/linker.lf

@@ -3,17 +3,17 @@ archive: libapp_trace.a
 entries:
     app_trace (noflash)
     app_trace_util (noflash)
-    if SYSVIEW_ENABLE = y:
+    if APPTRACE_SV_ENABLE = y:
       SEGGER_SYSVIEW (noflash)
-      SEGGER_RTT_esp32 (noflash)
+      SEGGER_RTT_esp (noflash)
       SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
       SEGGER_SYSVIEW_FreeRTOS (noflash)
 
 [mapping:app_trace_driver]
 archive: libdriver.a
 entries:
-    if SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y
-       || SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y:
+    if APPTRACE_SV_TS_SOURCE_TIMER_00 = y || APPTRACE_SV_TS_SOURCE_TIMER_01 = y
+       || APPTRACE_SV_TS_SOURCE_TIMER_10 = y || APPTRACE_SV_TS_SOURCE_TIMER_11 = y:
       timer (noflash)
     else:
       * (default)

components/app_trace/port/include/esp_app_trace_port.h

@@ -0,0 +1,43 @@
+// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef ESP_APP_TRACE_PORT_H_
+#define ESP_APP_TRACE_PORT_H_
+
+#include "esp_app_trace_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Apptrace HW interface. */
+typedef struct {
+    esp_err_t (*init)(void *hw_data);
+    uint8_t *(*get_up_buffer)(void *hw_data, uint32_t, esp_apptrace_tmo_t *);
+    esp_err_t (*put_up_buffer)(void *hw_data, uint8_t *, esp_apptrace_tmo_t *);
+    esp_err_t (*flush_up_buffer_nolock)(void *hw_data, uint32_t, esp_apptrace_tmo_t *);
+    esp_err_t (*flush_up_buffer)(void *hw_data, esp_apptrace_tmo_t *);
+    void (*down_buffer_config)(void *hw_data, uint8_t *buf, uint32_t size);
+    uint8_t *(*get_down_buffer)(void *hw_data, uint32_t *, esp_apptrace_tmo_t *);
+    esp_err_t (*put_down_buffer)(void *hw_data, uint8_t *, esp_apptrace_tmo_t *);
+    bool (*host_is_connected)(void *hw_data);
+} esp_apptrace_hw_t;
+
+esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data);
+esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

components/app_trace/port/riscv/port.c

@@ -0,0 +1,374 @@
+#include "esp_log.h"
+#include "esp_app_trace_membufs_proto.h"
+#include "esp_app_trace_port.h"
+
+/** RISCV HW transport data */
+typedef struct {
+    uint8_t                             inited; // initialization state flags for every core
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_apptrace_lock_t                 lock;   // sync lock
+#endif
+    esp_apptrace_membufs_proto_data_t   membufs;
+} esp_apptrace_riscv_data_t;
+
+/** RISCV memory host iface control block */
+typedef struct {
+    uint32_t                    ctrl;
+    // - Guard field. If this register is not zero then CPU is changing this struct and
+    //   this guard field holds address of the instruction which application will execute when CPU finishes with those modifications.
+    uint32_t                    stat;
+    esp_apptrace_mem_block_t *  mem_blocks;
+} esp_apptrace_riscv_ctrl_block_t;
+
+#define RISCV_APPTRACE_SYSNR    0x64
+
+#define ESP_APPTRACE_RISCV_BLOCK_LEN_MSK         0x7FFFUL
+#define ESP_APPTRACE_RISCV_BLOCK_LEN(_l_)        ((_l_) & ESP_APPTRACE_RISCV_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_RISCV_BLOCK_LEN_GET(_v_)    ((_v_) & ESP_APPTRACE_RISCV_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_RISCV_BLOCK_ID_MSK          0x7FUL
+#define ESP_APPTRACE_RISCV_BLOCK_ID(_id_)        (((_id_) & ESP_APPTRACE_RISCV_BLOCK_ID_MSK) << 15)
+#define ESP_APPTRACE_RISCV_BLOCK_ID_GET(_v_)     (((_v_) >> 15) & ESP_APPTRACE_RISCV_BLOCK_ID_MSK)
+#define ESP_APPTRACE_RISCV_HOST_DATA             (1 << 22)
+#define ESP_APPTRACE_RISCV_HOST_CONNECT          (1 << 23)
+
+#define ESP_APPTRACE_RISCV_INITED(_hw_)          ((_hw_)->inited & (1 << 0/*cpu_hal_get_core_id()*/))
+
+static esp_err_t esp_apptrace_riscv_init(esp_apptrace_riscv_data_t *hw_data);
+static esp_err_t esp_apptrace_riscv_flush(esp_apptrace_riscv_data_t *hw_data, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_riscv_flush_nolock(esp_apptrace_riscv_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+static uint8_t *esp_apptrace_riscv_up_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_riscv_up_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static void esp_apptrace_riscv_down_buffer_config(esp_apptrace_riscv_data_t *hw_data, uint8_t *buf, uint32_t size);
+static uint8_t *esp_apptrace_riscv_down_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_riscv_down_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static bool esp_apptrace_riscv_host_is_connected(esp_apptrace_riscv_data_t *hw_data);
+static esp_err_t esp_apptrace_riscv_buffer_swap_start(uint32_t curr_block_id);
+static esp_err_t esp_apptrace_riscv_buffer_swap(uint32_t new_block_id);
+static esp_err_t esp_apptrace_riscv_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len);
+static bool esp_apptrace_riscv_host_data_pending(void);
+
+
+const static char *TAG = "esp_apptrace";
+
+static esp_apptrace_riscv_ctrl_block_t s_tracing_ctrl[portNUM_PROCESSORS];
+
+esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data)
+{
+    return NULL;
+}
+
+esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
+{
+#if CONFIG_APPTRACE_DEST_JTAG
+    static esp_apptrace_membufs_proto_hw_t s_trace_proto_hw = {
+        .swap_start = esp_apptrace_riscv_buffer_swap_start,
+        .swap = esp_apptrace_riscv_buffer_swap,
+        .swap_end = esp_apptrace_riscv_buffer_swap_end,
+        .host_data_pending = esp_apptrace_riscv_host_data_pending,
+    };
+    static esp_apptrace_riscv_data_t s_trace_hw_data = {
+        .membufs = {
+            .hw = &s_trace_proto_hw,
+        },
+    };
+    static esp_apptrace_hw_t s_trace_hw = {
+        .init = (esp_err_t (*)(void *))esp_apptrace_riscv_init,
+        .get_up_buffer = (uint8_t *(*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_riscv_up_buffer_get,
+        .put_up_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_riscv_up_buffer_put,
+        .flush_up_buffer_nolock = (esp_err_t (*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_riscv_flush_nolock,
+        .flush_up_buffer = (esp_err_t (*)(void *, esp_apptrace_tmo_t *))esp_apptrace_riscv_flush,
+        .down_buffer_config = (void (*)(void *, uint8_t *, uint32_t ))esp_apptrace_riscv_down_buffer_config,
+        .get_down_buffer = (uint8_t *(*)(void *, uint32_t *, esp_apptrace_tmo_t *))esp_apptrace_riscv_down_buffer_get,
+        .put_down_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_riscv_down_buffer_put,
+        .host_is_connected = (bool (*)(void *))esp_apptrace_riscv_host_is_connected,
+    };
+    *data = &s_trace_hw_data;
+    return &s_trace_hw;
+#else
+    return NULL;
+#endif
+}
+
+/* Advertises apptrace control block address to host.
+   This function can be overriden with custom implementation,
+   e.g. OpenOCD flasher stub use own implementation of it. */
+__attribute__((weak)) int esp_apptrace_advertise_ctrl_block(void *ctrl_block_addr)
+{
+    register int sys_nr = RISCV_APPTRACE_SYSNR;
+    register int host_ret = 0;
+
+    if (!esp_cpu_in_ocd_debug_mode()) {
+        return 0;
+    }
+    __asm__ volatile ( \
+        ".option push\n" \
+        ".option norvc\n" \
+        "mv a0, %[sys_nr]\n" \
+        "mv a1, %[arg1]\n" \
+        "slli    zero,zero,0x1f\n" \
+        "ebreak\n" \
+        "srai    zero,zero,0x7\n" \
+        "mv %[host_ret], a0\n" \
+    	".option pop\n" \
+        :[host_ret]"=r"(host_ret)
+        :[sys_nr]"r"(sys_nr),[arg1]"r"(ctrl_block_addr):"a0","a1");
+    return host_ret;
+}
+
+/* Returns up buffers config.
+   This function can be overriden with custom implementation,
+   e.g. OpenOCD flasher stub use own implementation of it. */
+__attribute__((weak)) void esp_apptrace_get_up_buffers(esp_apptrace_mem_block_t mem_blocks_cfg[2])
+{
+    static uint8_t s_mem_blocks[2][CONFIG_APPTRACE_BUF_SIZE];
+
+    mem_blocks_cfg[0].start = s_mem_blocks[0];
+    mem_blocks_cfg[0].sz = CONFIG_APPTRACE_BUF_SIZE;
+    mem_blocks_cfg[1].start = s_mem_blocks[1];
+    mem_blocks_cfg[1].sz = CONFIG_APPTRACE_BUF_SIZE;
+}
+
+static esp_err_t esp_apptrace_riscv_lock(esp_apptrace_riscv_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_err_t ret = esp_apptrace_lock_take(&hw_data->lock, tmo);
+    if (ret != ESP_OK) {
+        return ESP_FAIL;
+    }
+#endif
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_riscv_unlock(esp_apptrace_riscv_data_t *hw_data)
+{
+    esp_err_t ret = ESP_OK;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    ret = esp_apptrace_lock_give(&hw_data->lock);
+#endif
+    return ret;
+}
+
+/*****************************************************************************************/
+/***************************** Apptrace HW iface *****************************************/
+/*****************************************************************************************/
+
+static esp_err_t esp_apptrace_riscv_init(esp_apptrace_riscv_data_t *hw_data)
+{
+    int core_id = cpu_hal_get_core_id();
+
+    if (hw_data->inited == 0) {
+        esp_apptrace_mem_block_t mem_blocks_cfg[2];
+        esp_apptrace_get_up_buffers(mem_blocks_cfg);
+        esp_err_t res = esp_apptrace_membufs_init(&hw_data->membufs, mem_blocks_cfg);
+        if (res != ESP_OK) {
+            ESP_APPTRACE_LOGE("Failed to init membufs proto (%d)!", res);
+            return res;
+        }
+#if CONFIG_APPTRACE_LOCK_ENABLE
+        esp_apptrace_lock_init(&hw_data->lock);
+#endif
+    }
+    hw_data->inited |= 1 << core_id;
+    ESP_APPTRACE_LOGI("Apptrace initialized on CPU%d. Tracing control block @ %p.", core_id, &s_tracing_ctrl[core_id]);
+    s_tracing_ctrl[core_id].mem_blocks = hw_data->membufs.blocks;
+    for (int i = 0; i < 2; i++) {
+        ESP_APPTRACE_LOGD("Mem buf[%d] %d bytes @ %p (%p/%p)", i,
+            s_tracing_ctrl[core_id].mem_blocks[i].sz, s_tracing_ctrl[core_id].mem_blocks[i].start,
+            &(s_tracing_ctrl[core_id].mem_blocks[i].start), &(s_tracing_ctrl[core_id].mem_blocks[i].sz));
+    }
+    // notify host about control block address
+    int res = esp_apptrace_advertise_ctrl_block(&s_tracing_ctrl[core_id]);
+    assert(res == 0 && "Falied to send config to host!");
+
+    return ESP_OK;
+}
+
+static uint8_t *esp_apptrace_riscv_up_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_up_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_riscv_up_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_up_buffer_put() just modifies buffer's header
+    esp_err_t res = esp_apptrace_membufs_up_buffer_put(&hw_data->membufs, ptr, tmo);
+    return res;
+}
+
+static void esp_apptrace_riscv_down_buffer_config(esp_apptrace_riscv_data_t *hw_data, uint8_t *buf, uint32_t size)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return;
+    }
+    esp_apptrace_membufs_down_buffer_config(&hw_data->membufs, buf, size);
+}
+
+static uint8_t *esp_apptrace_riscv_down_buffer_get(esp_apptrace_riscv_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_down_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_riscv_down_buffer_put(esp_apptrace_riscv_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_down_buffer_put() does nothing
+    /*esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }*/
+
+    esp_err_t res = esp_apptrace_membufs_down_buffer_put(&hw_data->membufs, ptr, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    /*if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }*/
+    return res;
+}
+
+static bool esp_apptrace_riscv_host_is_connected(esp_apptrace_riscv_data_t *hw_data)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return false;
+    }
+    return s_tracing_ctrl[cpu_hal_get_core_id()].ctrl & ESP_APPTRACE_RISCV_HOST_CONNECT ? true : false;
+}
+
+static esp_err_t esp_apptrace_riscv_flush_nolock(esp_apptrace_riscv_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    return esp_apptrace_membufs_flush_nolock(&hw_data->membufs, min_sz, tmo);
+}
+
+static esp_err_t esp_apptrace_riscv_flush(esp_apptrace_riscv_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_RISCV_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    esp_err_t res = esp_apptrace_riscv_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }
+
+    res = esp_apptrace_membufs_flush_nolock(&hw_data->membufs, 0, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_riscv_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return res;
+}
+
+/*****************************************************************************************/
+/************************** Membufs proto HW iface ***************************************/
+/*****************************************************************************************/
+
+static inline void esp_apptrace_riscv_buffer_swap_lock(void)
+{
+    extern uint32_t __esp_apptrace_riscv_updated;
+
+    // indicate to host that we are about to update.
+    // this is used only to place CPU into streaming mode at tracing startup
+    // before starting streaming host can halt us after we read  ESP_APPTRACE_RISCV_CTRL_REG and before we updated it
+    // HACK: in this case host will set breakpoint just after ESP_APPTRACE_RISCV_CTRL_REG update,
+    // here we set address to set bp at
+    // enter ERI update critical section
+    s_tracing_ctrl[cpu_hal_get_core_id()].stat = (uint32_t)&__esp_apptrace_riscv_updated;
+}
+
+static __attribute__((noinline)) void esp_apptrace_riscv_buffer_swap_unlock(void)
+{
+    // exit ERI update critical section
+    s_tracing_ctrl[cpu_hal_get_core_id()].stat = 0;
+    // TODO: currently host sets breakpoint, use break instruction to stop;
+    // it will allow to use ESP_APPTRACE_RISCV_STAT_REG for other purposes
+    asm volatile (
+        "    .global     __esp_apptrace_riscv_updated\n"
+        "__esp_apptrace_riscv_updated:\n"); // host will set bp here to resolve collision at streaming start
+}
+
+static esp_err_t esp_apptrace_riscv_buffer_swap_start(uint32_t curr_block_id)
+{
+    esp_err_t res = ESP_OK;
+
+    esp_apptrace_riscv_buffer_swap_lock();
+
+    uint32_t ctrl_reg = s_tracing_ctrl[cpu_hal_get_core_id()].ctrl;
+    uint32_t host_connected = ESP_APPTRACE_RISCV_HOST_CONNECT & ctrl_reg;
+    if (host_connected) {
+        uint32_t acked_block = ESP_APPTRACE_RISCV_BLOCK_ID_GET(ctrl_reg);
+        uint32_t host_to_read = ESP_APPTRACE_RISCV_BLOCK_LEN_GET(ctrl_reg);
+        if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK)) {
+            ESP_APPTRACE_LOGD("[%d]: Can not switch %x %d %x %x/%lx", cpu_hal_get_core_id(), ctrl_reg, host_to_read, acked_block,
+                curr_block_id & ESP_APPTRACE_RISCV_BLOCK_ID_MSK, curr_block_id);
+            res = ESP_ERR_NO_MEM;
+            goto _on_err;
+        }
+    }
+    return ESP_OK;
+_on_err:
+    esp_apptrace_riscv_buffer_swap_unlock();
+    return res;
+}
+
+static esp_err_t esp_apptrace_riscv_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len)
+{
+    uint32_t ctrl_reg = s_tracing_ctrl[cpu_hal_get_core_id()].ctrl;
+    uint32_t host_connected = ESP_APPTRACE_RISCV_HOST_CONNECT & ctrl_reg;
+    s_tracing_ctrl[cpu_hal_get_core_id()].ctrl = ESP_APPTRACE_RISCV_BLOCK_ID(new_block_id) |
+              host_connected | ESP_APPTRACE_RISCV_BLOCK_LEN(prev_block_len);
+    esp_apptrace_riscv_buffer_swap_unlock();
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_riscv_buffer_swap(uint32_t new_block_id)
+{
+    /* do nothing */
+    return ESP_OK;
+}
+
+static bool esp_apptrace_riscv_host_data_pending(void)
+{
+    uint32_t ctrl_reg = s_tracing_ctrl[cpu_hal_get_core_id()].ctrl;
+    // ESP_APPTRACE_LOGV("%s() 0x%x", __func__, ctrl_reg);
+    return (ctrl_reg & ESP_APPTRACE_RISCV_HOST_DATA) ? true : false;
+}

components/app_trace/port/xtensa/port.c

@@ -0,0 +1,543 @@
+//
+// How It Works
+// ************
+
+// 1. Components Overview
+// ======================
+
+// Xtensa has useful feature: TRAX debug module. It allows recording program execution flow at run-time without disturbing CPU.
+// Exectution flow data are written to configurable Trace RAM block. Besides accessing Trace RAM itself TRAX module also allows to read/write
+// trace memory via its registers by means of JTAG, APB or ERI transactions.
+// ESP32 has two Xtensa cores with separate TRAX modules on them and provides two special memory regions to be used as trace memory.
+// Chip allows muxing access to those trace memory blocks in such a way that while one block is accessed by CPUs another one can be accessed by host
+// by means of reading/writing TRAX registers via JTAG. Blocks muxing is configurable at run-time and allows switching trace memory blocks between
+// accessors in round-robin fashion so they can read/write separate memory blocks without disturbing each other.
+// This module implements application tracing feature based on above mechanisms. It allows to transfer arbitrary user data to/from
+// host via JTAG with minimal impact on system performance. This module is implied to be used in the following tracing scheme.
+
+//                                                        ------>------                                         ----- (host components) -----
+//                                                        |           |                                         |                           |
+// -------------------   -----------------------     -----------------------     ----------------    ------     ---------   -----------------
+// |trace data source|-->|target tracing module|<--->|TRAX_MEM0 | TRAX_MEM1|---->|TRAX_DATA_REGS|<-->|JTAG|<--->|OpenOCD|-->|trace data sink|
+// -------------------   -----------------------     -----------------------     ----------------    ------     ---------   -----------------
+//                                 |                      |           |                                |
+//                                 |                      ------<------          ----------------      |
+//                                 |<------------------------------------------->|TRAX_CTRL_REGS|<---->|
+//                                                                               ----------------
+
+// In general tracing goes in the following way. User application requests tracing module to send some data by calling esp_apptrace_buffer_get(),
+// module allocates necessary buffer in current input trace block. Then user fills received buffer with data and calls esp_apptrace_buffer_put().
+// When current input trace block is filled with app data it is exposed to host and the second block becomes input one and buffer filling restarts.
+// While target application fills one TRAX block host reads another one via JTAG.
+// This module also allows communication in the opposite direction: from host to target. As it was said ESP32 and host can access different TRAX blocks
+// simultaneously, so while target writes trace data to one block host can write its own data (e.g. tracing commands) to another one then when
+// blocks are switched host receives trace data and target receives data written by host application. Target user application can read host data
+// by calling esp_apptrace_read() API.
+// To control buffer switching and for other communication purposes this implementation uses some TRAX registers. It is safe since HW TRAX tracing
+// can not be used along with application tracing feature so these registers are freely readable/writeable via JTAG from host and via ERI from ESP32 cores.
+// Overhead of this implementation on target CPU is produced only by allocating/managing buffers and copying of data.
+// On the host side special OpenOCD command must be used to read trace data.
+
+// 2. TRAX Registers layout
+// ========================
+
+// This module uses two TRAX HW registers to communicate with host SW (OpenOCD).
+//  - Control register uses TRAX_DELAYCNT as storage. Only lower 24 bits of TRAX_DELAYCNT are writable. Control register has the following bitfields:
+//   | 31..XXXXXX..24 | 23 .(host_connect). 23| 22..(block_id)..15 | 14..(block_len)..0 |
+//    14..0  bits - actual length of user data in trace memory block. Target updates it every time it fills memory block and exposes it to host.
+//                  Host writes zero to this field when it finishes reading exposed block;
+//    21..15 bits - trace memory block transfer ID. Block counter. It can overflow. Updated by target, host should not modify it. Actually can be 2 bits;
+//    22     bit  - 'host data present' flag. If set to one there is data from host, otherwise - no host data;
+//    23     bit  - 'host connected' flag. If zero then host is not connected and tracing module works in post-mortem mode, otherwise in streaming mode;
+// - Status register uses TRAX_TRIGGERPC as storage. If this register is not zero then current CPU is changing TRAX registers and
+//   this register holds address of the instruction which application will execute when it finishes with those registers modifications.
+//   See 'Targets Connection' setion for details.
+
+// 3. Modes of operation
+// =====================
+
+// This module supports two modes of operation:
+//  - Post-mortem mode. This is the default mode. In this mode application tracing module does not check whether host has read all the data from block
+//    exposed to it and switches block in any case. The mode does not need host interaction for operation and so can be useful when only the latest
+//    trace data are necessary, e.g. for analyzing crashes. On panic the latest data from current input block are exposed to host and host can read them.
+//    It can happen that system panic occurs when there are very small amount of data which are not exposed to host yet (e.g. crash just after the
+//    TRAX block switch). In this case the previous 16KB of collected data will be dropped and host will see the latest, but very small piece of trace.
+//    It can be insufficient to diagnose the problem. To avoid such situations there is menuconfig option
+//    CONFIG_APPTRACE_POSTMORTEM_FLUSH_THRESH
+//    which controls the threshold for flushing data in case of panic.
+//  - Streaming mode. Tracing module enters this mode when host connects to target and sets respective bits in control registers (per core).
+//    In this mode before switching the block tracing module waits for the host to read all the data from the previously exposed block.
+//    On panic tracing module also waits (timeout is configured via menuconfig via CONFIG_APPTRACE_ONPANIC_HOST_FLUSH_TMO) for the host to read all data.
+
+// 4. Communication Protocol
+// =========================
+
+// 4.1 Trace Memory Blocks
+// -----------------------
+
+// Communication is controlled via special register. Host periodically polls control register on each core to find out if there are any data available.
+// When current input memory block is filled it is exposed to host and 'block_len' and 'block_id' fields are updated in the control register.
+// Host reads new register value and according to it's value starts reading data from exposed block. Meanwhile target starts filling another trace block.
+// When host finishes reading the block it clears 'block_len' field in control register indicating to the target that it is ready to accept the next one.
+// If the host has some data to transfer to the target it writes them to trace memory block before clearing 'block_len' field. Then it sets
+// 'host_data_present' bit and clears 'block_len' field in control register. Upon every block switch target checks 'host_data_present' bit and if it is set
+// reads them to down buffer before writing any trace data to switched TRAX block.
+
+// 4.2 User Data Chunks Level
+// --------------------------
+
+// Since trace memory block is shared between user data chunks and data copying is performed on behalf of the API user (in its normal context) in
+// multithreading environment it can happen that task/ISR which copies data is preempted by another high prio task/ISR. So it is possible situation
+// that task/ISR will fail to complete filling its data chunk before the whole trace block is exposed to the host. To handle such conditions tracing
+// module prepends all user data chunks with header which contains allocated buffer size and actual data length within it. OpenOCD command
+// which reads application traces reports error when it reads incomplete user data block.
+// Data which are transffered from host to target are also prepended with a header. Down channel data header is simple and consists of one two bytes field
+// containing length of host data following the header.
+
+// 4.3 Data Buffering
+// ------------------
+
+// It takes some time for the host to read TRAX memory block via JTAG. In streaming mode it can happen that target has filled its TRAX block, but host
+// has not completed reading of the previous one yet. So in this case time critical tracing calls (which can not be delayed for too long time due to
+// the lack of free memory in TRAX block) can be dropped. To avoid such scenarios tracing module implements data buffering. Buffered data will be sent
+// to the host later when TRAX block switch occurs. The maximum size of the buffered data is controlled by menuconfig option
+// CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX.
+
+// 4.4 Target Connection/Disconnection
+// -----------------------------------
+
+// When host is going to start tracing in streaming mode it needs to put both ESP32 cores into initial state when 'host connected' bit is set
+// on both cores. To accomplish this host halts both cores and sets this bit in TRAX registers. But target code can be halted in state when it has read control
+// register but has not updated its value. To handle such situations target code indicates to the host that it is updating control register by writing
+// non-zero value to status register. Actually it writes address of the instruction which it will execute when it finishes with
+// the registers update. When target is halted during control register update host sets breakpoint at the address from status register and resumes CPU.
+// After target code finishes with register update it is halted on breakpoint, host detects it and safely sets 'host connected' bit. When both cores
+// are set up they are resumed. Tracing starts without further intrusion into CPUs work.
+// When host is going to stop tracing in streaming mode it needs to disconnect targets. Disconnection process is done using the same algorithm
+// as for connecting, but 'host connected' bits are cleared on ESP32 cores.
+
+// 5. Module Access Synchronization
+// ================================
+
+// Access to internal module's data is synchronized with custom mutex. Mutex is a wrapper for portMUX_TYPE and uses almost the same sync mechanism as in
+// vPortCPUAcquireMutex/vPortCPUReleaseMutex. The mechanism uses S32C1I Xtensa instruction to implement exclusive access to module's data from tasks and
+// ISRs running on both cores. Also custom mutex allows specifying timeout for locking operation. Locking routine checks underlaying mutex in cycle until
+// it gets its ownership or timeout expires. The differences of application tracing module's mutex implementation from vPortCPUAcquireMutex/vPortCPUReleaseMutex are:
+// - Support for timeouts.
+// - Local IRQs for CPU which owns the mutex are disabled till the call to unlocking routine. This is made to avoid possible task's prio inversion.
+//   When low prio task takes mutex and enables local IRQs gets preempted by high prio task which in its turn can try to acquire mutex using infinite timeout.
+//   So no local task switch occurs when mutex is locked. But this does not apply to tasks on another CPU.
+//   WARNING: Priority inversion can happen when low prio task works on one CPU and medium and high prio tasks work on another.
+// WARNING: Care must be taken when selecting timeout values for trace calls from ISRs. Tracing module does not care about watchdogs when waiting
+// on internal locks and for host to complete previous block reading, so if timeout value exceeds watchdog's one it can lead to the system reboot.
+
+// 6. Timeouts
+// ===========
+
+// Timeout mechanism is based on xthal_get_ccount() routine and supports timeout values in microseconds.
+// There are two situations when task/ISR can be delayed by tracing API call. Timeout mechanism takes into account both conditions:
+// - Trace data are locked by another task/ISR. When wating on trace data lock.
+// - Current TRAX memory input block is full when working in streaming mode (host is connected). When waiting for host to complete previous block reading.
+// When wating for any of above conditions xthal_get_ccount() is called periodically to calculate time elapsed from trace API routine entry. When elapsed
+// time exceeds specified timeout value operation is canceled and ESP_ERR_TIMEOUT code is returned.
+#include "sdkconfig.h"
+#include "soc/soc.h"
+#include "soc/dport_access.h"
+#if CONFIG_IDF_TARGET_ESP32
+#include "soc/dport_reg.h"
+#elif CONFIG_IDF_TARGET_ESP32S2
+#include "soc/sensitive_reg.h"
+#endif
+#include "eri.h"
+#include "trax.h"
+#include "esp_log.h"
+#include "esp_app_trace_membufs_proto.h"
+#include "esp_app_trace_port.h"
+
+// TODO: move these (and same definitions in trax.c to dport_reg.h)
+#if CONFIG_IDF_TARGET_ESP32
+#define TRACEMEM_MUX_PROBLK0_APPBLK1            0
+#define TRACEMEM_MUX_BLK0_ONLY                  1
+#define TRACEMEM_MUX_BLK1_ONLY                  2
+#define TRACEMEM_MUX_PROBLK1_APPBLK0            3
+#elif CONFIG_IDF_TARGET_ESP32S2
+#define TRACEMEM_MUX_BLK0_NUM                   19
+#define TRACEMEM_MUX_BLK1_NUM                   20
+#define TRACEMEM_BLK_NUM2ADDR(_n_)              (0x3FFB8000UL + 0x4000UL*((_n_)-4))
+#endif
+
+// TRAX is disabled, so we use its registers for our own purposes
+// | 31..XXXXXX..24 | 23 .(host_connect). 23 | 22 .(host_data). 22| 21..(block_id)..15 | 14..(block_len)..0 |
+#define ESP_APPTRACE_TRAX_CTRL_REG              ERI_TRAX_DELAYCNT
+#define ESP_APPTRACE_TRAX_STAT_REG              ERI_TRAX_TRIGGERPC
+
+#define ESP_APPTRACE_TRAX_BLOCK_LEN_MSK         0x7FFFUL
+#define ESP_APPTRACE_TRAX_BLOCK_LEN(_l_)        ((_l_) & ESP_APPTRACE_TRAX_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_TRAX_BLOCK_LEN_GET(_v_)    ((_v_) & ESP_APPTRACE_TRAX_BLOCK_LEN_MSK)
+#define ESP_APPTRACE_TRAX_BLOCK_ID_MSK          0x7FUL
+#define ESP_APPTRACE_TRAX_BLOCK_ID(_id_)        (((_id_) & ESP_APPTRACE_TRAX_BLOCK_ID_MSK) << 15)
+#define ESP_APPTRACE_TRAX_BLOCK_ID_GET(_v_)     (((_v_) >> 15) & ESP_APPTRACE_TRAX_BLOCK_ID_MSK)
+#define ESP_APPTRACE_TRAX_HOST_DATA             (1 << 22)
+#define ESP_APPTRACE_TRAX_HOST_CONNECT          (1 << 23)
+
+#define ESP_APPTRACE_TRAX_INITED(_hw_)          ((_hw_)->inited & (1 << cpu_hal_get_core_id()))
+
+#if CONFIG_IDF_TARGET_ESP32
+static uint8_t * const s_trax_blocks[] = {
+    (uint8_t *) 0x3FFFC000,
+    (uint8_t *) 0x3FFF8000
+};
+#elif CONFIG_IDF_TARGET_ESP32S2
+static uint8_t * const s_trax_blocks[] = {
+    (uint8_t *)TRACEMEM_BLK_NUM2ADDR(TRACEMEM_MUX_BLK0_NUM),
+    (uint8_t *)TRACEMEM_BLK_NUM2ADDR(TRACEMEM_MUX_BLK1_NUM)
+};
+#endif
+
+#define ESP_APPTRACE_TRAX_BLOCK_SIZE            (0x4000UL)
+
+/** TRAX HW transport data */
+typedef struct {
+    uint8_t                             inited;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_apptrace_lock_t                 lock;   // sync lock
+#endif
+    esp_apptrace_membufs_proto_data_t   membufs;
+} esp_apptrace_trax_data_t;
+
+
+static esp_err_t esp_apptrace_trax_init(esp_apptrace_trax_data_t *hw_data);
+static esp_err_t esp_apptrace_trax_flush(esp_apptrace_trax_data_t *hw_data, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_trax_flush_nolock(esp_apptrace_trax_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+static uint8_t *esp_apptrace_trax_up_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_trax_up_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static void esp_apptrace_trax_down_buffer_config(esp_apptrace_trax_data_t *hw_data, uint8_t *buf, uint32_t size);
+static uint8_t *esp_apptrace_trax_down_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo);
+static esp_err_t esp_apptrace_trax_down_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+static bool esp_apptrace_trax_host_is_connected(esp_apptrace_trax_data_t *hw_data);
+static esp_err_t esp_apptrace_trax_buffer_swap_start(uint32_t curr_block_id);
+static esp_err_t esp_apptrace_trax_buffer_swap(uint32_t new_block_id);
+static esp_err_t esp_apptrace_trax_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len);
+static bool esp_apptrace_trax_host_data_pending(void);
+
+
+const static char *TAG = "esp_apptrace";
+
+esp_apptrace_hw_t *esp_apptrace_uart_hw_get(int num, void **data)
+{
+    return NULL;
+}
+
+esp_apptrace_hw_t *esp_apptrace_jtag_hw_get(void **data)
+{
+#if CONFIG_APPTRACE_DEST_JTAG
+    static esp_apptrace_membufs_proto_hw_t s_trax_proto_hw = {
+        .swap_start = esp_apptrace_trax_buffer_swap_start,
+        .swap = esp_apptrace_trax_buffer_swap,
+        .swap_end = esp_apptrace_trax_buffer_swap_end,
+        .host_data_pending = esp_apptrace_trax_host_data_pending,
+    };
+    static esp_apptrace_trax_data_t s_trax_hw_data = {
+        .membufs = {
+            .hw = &s_trax_proto_hw,
+        },
+    };
+    static esp_apptrace_hw_t s_trax_hw = {
+        .init = (esp_err_t (*)(void *))esp_apptrace_trax_init,
+        .get_up_buffer = (uint8_t *(*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_trax_up_buffer_get,
+        .put_up_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_trax_up_buffer_put,
+        .flush_up_buffer_nolock = (esp_err_t (*)(void *, uint32_t, esp_apptrace_tmo_t *))esp_apptrace_trax_flush_nolock,
+        .flush_up_buffer = (esp_err_t (*)(void *, esp_apptrace_tmo_t *))esp_apptrace_trax_flush,
+        .down_buffer_config = (void (*)(void *, uint8_t *, uint32_t ))esp_apptrace_trax_down_buffer_config,
+        .get_down_buffer = (uint8_t *(*)(void *, uint32_t *, esp_apptrace_tmo_t *))esp_apptrace_trax_down_buffer_get,
+        .put_down_buffer = (esp_err_t (*)(void *, uint8_t *, esp_apptrace_tmo_t *))esp_apptrace_trax_down_buffer_put,
+        .host_is_connected = (bool (*)(void *))esp_apptrace_trax_host_is_connected,
+    };
+    *data = &s_trax_hw_data;
+    return &s_trax_hw;
+#else
+    return NULL;
+#endif
+}
+
+static esp_err_t esp_apptrace_trax_lock(esp_apptrace_trax_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    esp_err_t ret = esp_apptrace_lock_take(&hw_data->lock, tmo);
+    if (ret != ESP_OK) {
+        return ESP_FAIL;
+    }
+#endif
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_trax_unlock(esp_apptrace_trax_data_t *hw_data)
+{
+    esp_err_t ret = ESP_OK;
+#if CONFIG_APPTRACE_LOCK_ENABLE
+    ret = esp_apptrace_lock_give(&hw_data->lock);
+#endif
+    return ret;
+}
+
+static inline void esp_apptrace_trax_hw_init(void)
+{
+    // Stop trace, if any (on the current CPU)
+    eri_write(ERI_TRAX_TRAXCTRL, TRAXCTRL_TRSTP);
+    eri_write(ERI_TRAX_TRAXCTRL, TRAXCTRL_TMEN);
+    eri_write(ESP_APPTRACE_TRAX_CTRL_REG, ESP_APPTRACE_TRAX_BLOCK_ID(0));
+    // this is for OpenOCD to let him know where stub entries vector is resided
+    // must be read by host before any transfer using TRAX
+    eri_write(ESP_APPTRACE_TRAX_STAT_REG, 0);
+
+    ESP_APPTRACE_LOGI("Initialized TRAX on CPU%d", cpu_hal_get_core_id());
+}
+
+static inline void esp_apptrace_trax_select_memory_block(int block_num)
+{
+    // select memory block to be exposed to the TRAX module (accessed by host)
+#if CONFIG_IDF_TARGET_ESP32
+    DPORT_WRITE_PERI_REG(DPORT_TRACEMEM_MUX_MODE_REG, block_num ? TRACEMEM_MUX_BLK0_ONLY : TRACEMEM_MUX_BLK1_ONLY);
+#elif CONFIG_IDF_TARGET_ESP32S2
+    WRITE_PERI_REG(DPORT_PMS_OCCUPY_3_REG, block_num ? BIT(TRACEMEM_MUX_BLK0_NUM-4) : BIT(TRACEMEM_MUX_BLK1_NUM-4));
+#endif
+}
+
+static inline void esp_apptrace_trax_memory_enable(void)
+{
+#if CONFIG_IDF_TARGET_ESP32
+        /* Enable trace memory on PRO CPU */
+        DPORT_WRITE_PERI_REG(DPORT_PRO_TRACEMEM_ENA_REG, DPORT_PRO_TRACEMEM_ENA_M);
+#if CONFIG_FREERTOS_UNICORE == 0
+        /* Enable trace memory on APP CPU */
+        DPORT_WRITE_PERI_REG(DPORT_APP_TRACEMEM_ENA_REG, DPORT_APP_TRACEMEM_ENA_M);
+#endif
+#endif
+}
+
+/*****************************************************************************************/
+/***************************** Apptrace HW iface *****************************************/
+/*****************************************************************************************/
+
+static esp_err_t esp_apptrace_trax_init(esp_apptrace_trax_data_t *hw_data)
+{
+    int core_id = cpu_hal_get_core_id();
+
+    // 'esp_apptrace_trax_init()' is called on every core, so ensure to do main initialization only once
+    if (core_id == 0) {
+        esp_apptrace_mem_block_t mem_blocks_cfg[2] = {
+            {
+                .start = s_trax_blocks[0],
+                .sz = ESP_APPTRACE_TRAX_BLOCK_SIZE
+            },
+            {
+                .start = s_trax_blocks[1],
+                .sz = ESP_APPTRACE_TRAX_BLOCK_SIZE
+            },
+        };
+        esp_err_t res = esp_apptrace_membufs_init(&hw_data->membufs, mem_blocks_cfg);
+        if (res != ESP_OK) {
+            ESP_APPTRACE_LOGE("Failed to init membufs proto (%d)!", res);
+            return res;
+        }
+#if CONFIG_APPTRACE_LOCK_ENABLE
+        esp_apptrace_lock_init(&hw_data->lock);
+#endif
+        esp_apptrace_trax_memory_enable();
+        esp_apptrace_trax_select_memory_block(0);
+    }
+    // init TRAX on this CPU
+    esp_apptrace_trax_hw_init();
+    hw_data->inited |= 1 << core_id;
+
+    return ESP_OK;
+}
+
+static uint8_t *esp_apptrace_trax_up_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_up_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_trax_up_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_up_buffer_put() just modifies buffer's header
+    esp_err_t res = esp_apptrace_membufs_up_buffer_put(&hw_data->membufs, ptr, tmo);
+    return res;
+}
+
+static void esp_apptrace_trax_down_buffer_config(esp_apptrace_trax_data_t *hw_data, uint8_t *buf, uint32_t size)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return;
+    }
+    esp_apptrace_membufs_down_buffer_config(&hw_data->membufs, buf, size);
+}
+
+static uint8_t *esp_apptrace_trax_down_buffer_get(esp_apptrace_trax_data_t *hw_data, uint32_t *size, esp_apptrace_tmo_t *tmo)
+{
+    uint8_t *ptr;
+
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return NULL;
+    }
+    esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return NULL;
+    }
+
+    ptr = esp_apptrace_membufs_down_buffer_get(&hw_data->membufs, size, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return ptr;
+}
+
+static esp_err_t esp_apptrace_trax_down_buffer_put(esp_apptrace_trax_data_t *hw_data, uint8_t *ptr, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    // Can avoid locking because esp_apptrace_membufs_down_buffer_put() does nothing
+    /*esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }*/
+
+    esp_err_t res = esp_apptrace_membufs_down_buffer_put(&hw_data->membufs, ptr, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    /*if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }*/
+    return res;
+}
+
+static bool esp_apptrace_trax_host_is_connected(esp_apptrace_trax_data_t *hw_data)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return false;
+    }
+    return eri_read(ESP_APPTRACE_TRAX_CTRL_REG) & ESP_APPTRACE_TRAX_HOST_CONNECT ? true : false;
+}
+
+static esp_err_t esp_apptrace_trax_flush_nolock(esp_apptrace_trax_data_t *hw_data, uint32_t min_sz, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    return esp_apptrace_membufs_flush_nolock(&hw_data->membufs, min_sz, tmo);
+}
+
+static esp_err_t esp_apptrace_trax_flush(esp_apptrace_trax_data_t *hw_data, esp_apptrace_tmo_t *tmo)
+{
+    if (!ESP_APPTRACE_TRAX_INITED(hw_data)) {
+        return ESP_ERR_INVALID_STATE;
+    }
+    esp_err_t res = esp_apptrace_trax_lock(hw_data, tmo);
+    if (res != ESP_OK) {
+        return res;
+    }
+
+    res = esp_apptrace_membufs_flush_nolock(&hw_data->membufs, 0, tmo);
+
+    // now we can safely unlock apptrace to allow other tasks/ISRs to get other buffers and write their data
+    if (esp_apptrace_trax_unlock(hw_data) != ESP_OK) {
+        assert(false && "Failed to unlock apptrace data!");
+    }
+    return res;
+}
+
+/*****************************************************************************************/
+/************************** Membufs proto HW iface ***************************************/
+/*****************************************************************************************/
+
+static inline void esp_apptrace_trax_buffer_swap_lock(void)
+{
+    extern uint32_t __esp_apptrace_trax_eri_updated;
+
+    // indicate to host that we are about to update.
+    // this is used only to place CPU into streaming mode at tracing startup
+    // before starting streaming host can halt us after we read  ESP_APPTRACE_TRAX_CTRL_REG and before we updated it
+    // HACK: in this case host will set breakpoint just after ESP_APPTRACE_TRAX_CTRL_REG update,
+    // here we set address to set bp at
+    // enter ERI update critical section
+    eri_write(ESP_APPTRACE_TRAX_STAT_REG, (uint32_t)&__esp_apptrace_trax_eri_updated);
+}
+
+static __attribute__((noinline)) void esp_apptrace_trax_buffer_swap_unlock(void)
+{
+    // exit ERI update critical section
+    eri_write(ESP_APPTRACE_TRAX_STAT_REG, 0x0);
+    // TODO: currently host sets breakpoint, use break instruction to stop;
+    // it will allow to use ESP_APPTRACE_TRAX_STAT_REG for other purposes
+    asm volatile (
+        "    .global     __esp_apptrace_trax_eri_updated\n"
+        "__esp_apptrace_trax_eri_updated:\n"); // host will set bp here to resolve collision at streaming start
+}
+
+static esp_err_t esp_apptrace_trax_buffer_swap_start(uint32_t curr_block_id)
+{
+    esp_err_t res = ESP_OK;
+
+    esp_apptrace_trax_buffer_swap_lock();
+
+    uint32_t ctrl_reg = eri_read(ESP_APPTRACE_TRAX_CTRL_REG);
+    uint32_t host_connected = ESP_APPTRACE_TRAX_HOST_CONNECT & ctrl_reg;
+    if (host_connected) {
+        uint32_t acked_block = ESP_APPTRACE_TRAX_BLOCK_ID_GET(ctrl_reg);
+        uint32_t host_to_read = ESP_APPTRACE_TRAX_BLOCK_LEN_GET(ctrl_reg);
+        if (host_to_read != 0 || acked_block != (curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK)) {
+            ESP_APPTRACE_LOGD("HC[%d]: Can not switch %x %d %x %x/%lx", cpu_hal_get_core_id(), ctrl_reg, host_to_read, acked_block,
+                curr_block_id & ESP_APPTRACE_TRAX_BLOCK_ID_MSK, curr_block_id);
+            res = ESP_ERR_NO_MEM;
+            goto _on_err;
+        }
+    }
+    return ESP_OK;
+_on_err:
+    esp_apptrace_trax_buffer_swap_unlock();
+    return res;
+}
+
+static esp_err_t esp_apptrace_trax_buffer_swap_end(uint32_t new_block_id, uint32_t prev_block_len)
+{
+    uint32_t ctrl_reg = eri_read(ESP_APPTRACE_TRAX_CTRL_REG);
+    uint32_t host_connected = ESP_APPTRACE_TRAX_HOST_CONNECT & ctrl_reg;
+    eri_write(ESP_APPTRACE_TRAX_CTRL_REG, ESP_APPTRACE_TRAX_BLOCK_ID(new_block_id) |
+              host_connected | ESP_APPTRACE_TRAX_BLOCK_LEN(prev_block_len));
+    esp_apptrace_trax_buffer_swap_unlock();
+    return ESP_OK;
+}
+
+static esp_err_t esp_apptrace_trax_buffer_swap(uint32_t new_block_id)
+{
+    esp_apptrace_trax_select_memory_block(new_block_id);
+    return ESP_OK;
+}
+
+static bool esp_apptrace_trax_host_data_pending(void)
+{
+    uint32_t ctrl_reg = eri_read(ESP_APPTRACE_TRAX_CTRL_REG);
+    return (ctrl_reg & ESP_APPTRACE_TRAX_HOST_DATA) ? true : false;
+}

components/app_trace/private_include/esp_app_trace_membufs_proto.h

@@ -0,0 +1,70 @@
+// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef ESP_APP_TRACE_MEMBUFS_PROTO_H_
+#define ESP_APP_TRACE_MEMBUFS_PROTO_H_
+
+#include "esp_app_trace_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** TRAX HW transport state */
+typedef struct {
+    uint32_t                   in_block;     // input block ID
+    // TODO: change to uint16_t
+    uint32_t                   markers[2];   // block filling level markers
+} esp_apptrace_membufs_state_t;
+
+/** memory block parameters,
+ * should be packed, because it is read from the host */
+typedef struct {
+    uint8_t   *start;   // start address
+    uint32_t   sz;      // size
+} esp_apptrace_mem_block_t;
+
+typedef struct {
+    esp_err_t (*swap_start)(uint32_t curr_block_id);
+    esp_err_t (*swap)(uint32_t new_block_id);
+    esp_err_t (*swap_end)(uint32_t new_block_id, uint32_t prev_block_len);
+    bool (*host_data_pending)(void);
+} esp_apptrace_membufs_proto_hw_t;
+
+typedef struct {
+    esp_apptrace_membufs_proto_hw_t *       hw;
+    volatile esp_apptrace_membufs_state_t   state;       // state
+    esp_apptrace_mem_block_t                blocks[2];   // memory blocks
+#if CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX > 0
+    // ring buffer control struct for pending user blocks
+    esp_apptrace_rb_t                   rb_pend;
+    // storage for pending user blocks
+    uint8_t                             pending_data[CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX + 1];
+#endif
+    // ring buffer control struct for data from host (down buffer)
+    esp_apptrace_rb_t                       rb_down;
+} esp_apptrace_membufs_proto_data_t;
+
+esp_err_t esp_apptrace_membufs_init(esp_apptrace_membufs_proto_data_t *proto, const esp_apptrace_mem_block_t blocks_cfg[2]);
+void esp_apptrace_membufs_down_buffer_config(esp_apptrace_membufs_proto_data_t *data, uint8_t *buf, uint32_t size);
+uint8_t *esp_apptrace_membufs_down_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t *size, esp_apptrace_tmo_t *tmo);
+esp_err_t esp_apptrace_membufs_down_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+uint8_t *esp_apptrace_membufs_up_buffer_get(esp_apptrace_membufs_proto_data_t *proto, uint32_t size, esp_apptrace_tmo_t *tmo);
+esp_err_t esp_apptrace_membufs_up_buffer_put(esp_apptrace_membufs_proto_data_t *proto, uint8_t *ptr, esp_apptrace_tmo_t *tmo);
+esp_err_t esp_apptrace_membufs_flush_nolock(esp_apptrace_membufs_proto_data_t *proto, uint32_t min_sz, esp_apptrace_tmo_t *tmo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

components/app_trace/sdkconfig.rename

@@ -3,10 +3,34 @@
 
 CONFIG_ESP32_APPTRACE_DESTINATION      					CONFIG_APPTRACE_DESTINATION
 CONFIG_ESP32_APPTRACE_DEST_NONE							CONFIG_APPTRACE_DEST_NONE
-CONFIG_ESP32_APPTRACE_DEST_TRAX							CONFIG_APPTRACE_DEST_TRAX
+CONFIG_ESP32_APPTRACE_DEST_TRAX							CONFIG_APPTRACE_DEST_JTAG
 CONFIG_ESP32_APPTRACE_ENABLE							CONFIG_APPTRACE_ENABLE
 CONFIG_ESP32_APPTRACE_LOCK_ENABLE						CONFIG_APPTRACE_LOCK_ENABLE
 CONFIG_ESP32_APPTRACE_ONPANIC_HOST_FLUSH_TMO			CONFIG_APPTRACE_ONPANIC_HOST_FLUSH_TMO
 CONFIG_ESP32_APPTRACE_POSTMORTEM_FLUSH_TRAX_THRESH      CONFIG_APPTRACE_POSTMORTEM_FLUSH_THRESH
 CONFIG_ESP32_APPTRACE_PENDING_DATA_SIZE_MAX				CONFIG_APPTRACE_PENDING_DATA_SIZE_MAX
 CONFIG_ESP32_GCOV_ENABLE								CONFIG_APPTRACE_GCOV_ENABLE
+
+CONFIG_SYSVIEW_ENABLE							        CONFIG_APPTRACE_SV_ENABLE
+CONFIG_SYSVIEW_TS_SOURCE							    CONFIG_APPTRACE_SV_TS_SOURCE
+CONFIG_SYSVIEW_TS_SOURCE_CCOUNT							CONFIG_APPTRACE_SV_TS_SOURCE_CCOUNT
+CONFIG_SYSVIEW_TS_SOURCE_TIMER_00						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_00
+CONFIG_SYSVIEW_TS_SOURCE_TIMER_01						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_01
+CONFIG_SYSVIEW_TS_SOURCE_TIMER_10						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_10
+CONFIG_SYSVIEW_TS_SOURCE_TIMER_11						CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_11
+CONFIG_SYSVIEW_TS_SOURCE_ESP_TIMER						CONFIG_APPTRACE_SV_TS_SOURCE_ESP_TIMER
+CONFIG_SYSVIEW_MAX_TASKS							    CONFIG_APPTRACE_SV_MAX_TASKS
+CONFIG_SYSVIEW_BUF_WAIT_TMO							    CONFIG_APPTRACE_SV_BUF_WAIT_TMO
+CONFIG_SYSVIEW_EVT_OVERFLOW_ENABLE						CONFIG_APPTRACE_SV_EVT_OVERFLOW_ENABLE
+CONFIG_SYSVIEW_EVT_ISR_ENTER_ENABLE						CONFIG_APPTRACE_SV_EVT_ISR_ENTER_ENABLE
+CONFIG_SYSVIEW_EVT_ISR_EXIT_ENABLE						CONFIG_APPTRACE_SV_EVT_ISR_EXIT_ENABLE
+CONFIG_SYSVIEW_EVT_ISR_TO_SCHEDULER_ENABLE				CONFIG_APPTRACE_SV_EVT_ISR_TO_SCHED_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_START_EXEC_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_START_EXEC_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_STOP_EXEC_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_STOP_EXEC_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_START_READY_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_START_READY_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_STOP_READY_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_STOP_READY_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_CREATE_ENABLE					CONFIG_APPTRACE_SV_EVT_TASK_CREATE_ENABLE
+CONFIG_SYSVIEW_EVT_TASK_TERMINATE_ENABLE				CONFIG_APPTRACE_SV_EVT_TASK_TERMINATE_ENABLE
+CONFIG_SYSVIEW_EVT_IDLE_ENABLE							CONFIG_APPTRACE_SV_EVT_IDLE_ENABLE
+CONFIG_SYSVIEW_EVT_TIMER_ENTER_ENABLE					CONFIG_APPTRACE_SV_EVT_TIMER_ENTER_ENABLE
+CONFIG_SYSVIEW_EVT_TIMER_EXIT_ENABLE					CONFIG_APPTRACE_SV_EVT_TIMER_EXIT_ENABLE

components/app_trace/sys_view/SEGGER/SEGGER_RTT.h

@@ -159,8 +159,8 @@ unsigned     SEGGER_RTT_WriteNoLock             (unsigned BufferIndex, const voi
 unsigned     SEGGER_RTT_WriteSkipNoLock         (unsigned BufferIndex, const void* pBuffer, unsigned NumBytes);
 unsigned     SEGGER_RTT_WriteString             (unsigned BufferIndex, const char* s);
 void         SEGGER_RTT_WriteWithOverwriteNoLock(unsigned BufferIndex, const void* pBuffer, unsigned NumBytes);
-void         SEGGER_RTT_ESP32_FlushNoLock       (unsigned long min_sz, unsigned long tmo);
-void         SEGGER_RTT_ESP32_Flush             (unsigned long min_sz, unsigned long tmo);
+void         SEGGER_RTT_ESP_FlushNoLock       (unsigned long min_sz, unsigned long tmo);
+void         SEGGER_RTT_ESP_Flush             (unsigned long min_sz, unsigned long tmo);
 //
 // Function macro for performance optimization
 //

components/app_trace/sys_view/SEGGER/SEGGER_SYSVIEW.c

@@ -69,7 +69,7 @@ Additional information:
 
     Packets with IDs 24..31 are standard packets with extendible
     structure and contain a length field.
-    <ID><Lenght><Data><TimeStampDelta>
+    <ID><Length><Data><TimeStampDelta>
 
     Packets with IDs >= 32 always contain a length field.
     <ID><Length><Data><TimeStampDelta>

components/app_trace/sys_view/Sample/Config/SEGGER_SYSVIEW_Config_FreeRTOS.c

@@ -73,6 +73,8 @@ Revision: $Rev: 3734 $
 #include "esp32/clk.h"
 #elif CONFIG_IDF_TARGET_ESP32S2
 #include "esp32s2/clk.h"
+#elif CONFIG_IDF_TARGET_ESP32C3
+#include "esp32c3/clk.h"
 #endif
 
 
@@ -89,11 +91,17 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 
 // The target device name
 #define SYSVIEW_DEVICE_NAME     CONFIG_IDF_TARGET
+// The target core name
+#if CONFIG_IDF_TARGET_ARCH_XTENSA
+#define SYSVIEW_CORE_NAME       "xtensa"
+#elif CONFIG_IDF_TARGET_ARCH_RISCV
+#define SYSVIEW_CORE_NAME       "riscv"
+#endif
 
 // Determine which timer to use as timestamp source
-#if CONFIG_SYSVIEW_TS_SOURCE_CCOUNT
+#if CONFIG_APPTRACE_SV_TS_SOURCE_CCOUNT
 #define TS_USE_CCOUNT 1
-#elif CONFIG_SYSVIEW_TS_SOURCE_ESP_TIMER
+#elif CONFIG_APPTRACE_SV_TS_SOURCE_ESP_TIMER
 #define TS_USE_ESP_TIMER 1
 #else
 #define TS_USE_TIMERGROUP 1
@@ -109,13 +117,13 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 #define SYSVIEW_TIMESTAMP_FREQ  (esp_clk_apb_freq() / SYSVIEW_TIMER_DIV)
 
 // Timer ID and group ID
-#if defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_00) || defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_01)
+#if defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_00) || defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_10)
 #define TS_TIMER_ID 0
 #else
 #define TS_TIMER_ID 1
 #endif // TIMER_00 || TIMER_01
 
-#if defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_00) || defined(CONFIG_SYSVIEW_TS_SOURCE_TIMER_10)
+#if defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_00) || defined(CONFIG_APPTRACE_SV_TS_SOURCE_TIMER_01)
 #define TS_TIMER_GROUP 0
 #else
 #define TS_TIMER_GROUP 1
@@ -143,6 +151,7 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
 // The lowest RAM address used for IDs (pointers)
 #define SYSVIEW_RAM_BASE        (SOC_DROM_LOW)
 
+#ifdef CONFIG_FREERTOS_TICK_SUPPORT_CORETIMER
 #if CONFIG_FREERTOS_CORETIMER_0
     #define SYSTICK_INTR_ID (ETS_INTERNAL_TIMER0_INTR_SOURCE+ETS_INTERNAL_INTR_SOURCE_OFF)
 #endif
@@ -150,6 +159,10 @@ extern const SEGGER_SYSVIEW_OS_API SYSVIEW_X_OS_TraceAPI;
     #define SYSTICK_INTR_ID (ETS_INTERNAL_TIMER1_INTR_SOURCE+ETS_INTERNAL_INTR_SOURCE_OFF)
 #endif
 
+#elif CONFIG_FREERTOS_SYSTICK_USES_SYSTIMER
+    #define SYSTICK_INTR_ID (ETS_SYSTIMER_TARGET0_EDGE_INTR_SOURCE)
+#endif // CONFIG_FREERTOS_TICK_SUPPORT_CORETIMER
+
 // SystemView is single core specific: it implies that SEGGER_SYSVIEW_LOCK()
 // disables IRQs (disables rescheduling globally). So we can not use finite timeouts for locks and return error
 // in case of expiration, because error will not be handled and SEGGER's code will go further implying that
@@ -167,11 +180,13 @@ static esp_apptrace_lock_t s_sys_view_lock = {.mux = portMUX_INITIALIZER_UNLOCKE
 */
 static void _cbSendSystemDesc(void) {
     char irq_str[32];
-    SEGGER_SYSVIEW_SendSysDesc("N="SYSVIEW_APP_NAME",D="SYSVIEW_DEVICE_NAME",C=Xtensa,O=FreeRTOS");
+    SEGGER_SYSVIEW_SendSysDesc("N="SYSVIEW_APP_NAME",D="SYSVIEW_DEVICE_NAME",C="SYSVIEW_CORE_NAME",O=FreeRTOS");
     snprintf(irq_str, sizeof(irq_str), "I#%d=SysTick", SYSTICK_INTR_ID);
     SEGGER_SYSVIEW_SendSysDesc(irq_str);
     size_t isr_count = sizeof(esp_isr_names)/sizeof(esp_isr_names[0]);
     for (size_t i = 0; i < isr_count; ++i) {
+        if (esp_isr_names[i] == NULL || (ETS_INTERNAL_INTR_SOURCE_OFF + i) == SYSTICK_INTR_ID)
+            continue;
         snprintf(irq_str, sizeof(irq_str), "I#%d=%s", ETS_INTERNAL_INTR_SOURCE_OFF + i, esp_isr_names[i]);
         SEGGER_SYSVIEW_SendSysDesc(irq_str);
     }
@@ -213,43 +228,43 @@ void SEGGER_SYSVIEW_Conf(void) {
                         &SYSVIEW_X_OS_TraceAPI, _cbSendSystemDesc);
     SEGGER_SYSVIEW_SetRAMBase(SYSVIEW_RAM_BASE);
 
-#if !CONFIG_SYSVIEW_EVT_OVERFLOW_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_OVERFLOW_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_OVERFLOW;
 #endif
-#if !CONFIG_SYSVIEW_EVT_ISR_ENTER_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_ISR_ENTER_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_ISR_ENTER;
 #endif
-#if !CONFIG_SYSVIEW_EVT_ISR_EXIT_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_ISR_EXIT_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_ISR_EXIT;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_START_EXEC_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_START_EXEC_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_START_EXEC;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_STOP_EXEC_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_STOP_EXEC_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_STOP_EXEC;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_START_READY_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_START_READY_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_START_READY;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_STOP_READY_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_STOP_READY_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_STOP_READY;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_CREATE_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_CREATE_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_CREATE;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TASK_TERMINATE_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TASK_TERMINATE_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TASK_TERMINATE;
 #endif
-#if !CONFIG_SYSVIEW_EVT_IDLE_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_IDLE_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_IDLE;
 #endif
-#if !CONFIG_SYSVIEW_EVT_ISR_TO_SCHEDULER_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_ISR_TO_SCHED_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_ISR_TO_SCHEDULER;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TIMER_ENTER_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TIMER_ENTER_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TIMER_ENTER;
 #endif
-#if !CONFIG_SYSVIEW_EVT_TIMER_EXIT_ENABLE
+#if !CONFIG_APPTRACE_SV_EVT_TIMER_EXIT_ENABLE
     disable_evts |= SYSVIEW_EVTMASK_TIMER_EXIT;
 #endif
   SEGGER_SYSVIEW_DisableEvents(disable_evts);

components/app_trace/sys_view/Sample/OS/SEGGER_SYSVIEW_FreeRTOS.h

@@ -80,7 +80,7 @@ Notes:
   #define portSTACK_GROWTH              ( -1 )
 #endif
 
-#define SYSVIEW_FREERTOS_MAX_NOF_TASKS  CONFIG_SYSVIEW_MAX_TASKS
+#define SYSVIEW_FREERTOS_MAX_NOF_TASKS  CONFIG_APPTRACE_SV_MAX_TASKS
 
 /*********************************************************************
 *
@@ -209,7 +209,7 @@ Notes:
 #define apiID_VEVENTGROUPDELETE                   (72u)
 #define apiID_UXEVENTGROUPGETNUMBER               (73u)
 
-#define traceTASK_NOTIFY_TAKE()                                       SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_ULTASKNOTIFYTAKE, xClearCountOnExit, xTicksToWait)
+#define traceTASK_NOTIFY_TAKE( uxIndexToWait )                        SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_ULTASKNOTIFYTAKE, xClearCountOnExit, xTicksToWait)
 #define traceTASK_DELAY()                                             SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKDELAY, xTicksToDelay)
 #define traceTASK_DELAY_UNTIL()                                       SEGGER_SYSVIEW_RecordVoid(apiFastID_OFFSET + apiID_VTASKDELAYUNTIL)
 #define traceTASK_DELETE( pxTCB )                                     if (pxTCB != NULL) {                                              \
@@ -217,16 +217,16 @@ Notes:
 						                                                                      SEGGER_SYSVIEW_ShrinkId((U32)pxTCB)); 	\
 					                                                    SYSVIEW_DeleteTask((U32)pxTCB);                                 \
 					                                                  }
-#define traceTASK_NOTIFY_GIVE_FROM_ISR()                              SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_VTASKNOTIFYGIVEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), (U32)pxHigherPriorityTaskWoken)
+#define traceTASK_NOTIFY_GIVE_FROM_ISR( uxIndexToNotify )             SEGGER_SYSVIEW_RecordU32x2(apiFastID_OFFSET + apiID_VTASKNOTIFYGIVEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), (U32)pxHigherPriorityTaskWoken)
 #define traceTASK_PRIORITY_INHERIT( pxTCB, uxPriority )               SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKPRIORITYINHERIT, (U32)pxMutexHolder)
 #define traceTASK_RESUME( pxTCB )                                     SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKRESUME, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB))
 #define traceINCREASE_TICK_COUNT( xTicksToJump )                      SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKSTEPTICK, xTicksToJump)
 #define traceTASK_SUSPEND( pxTCB )                                    SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VTASKSUSPEND, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB))
 #define traceTASK_PRIORITY_DISINHERIT( pxTCB, uxBasePriority )        SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_XTASKPRIORITYDISINHERIT, (U32)pxMutexHolder)
 #define traceTASK_RESUME_FROM_ISR( pxTCB )                            SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_XTASKRESUMEFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB))
-#define traceTASK_NOTIFY()                                            SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFY, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue)
-#define traceTASK_NOTIFY_FROM_ISR()                                   SYSVIEW_RecordU32x5(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFYFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue, (U32)pxHigherPriorityTaskWoken)
-#define traceTASK_NOTIFY_WAIT()                                       SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKNOTIFYWAIT, ulBitsToClearOnEntry, ulBitsToClearOnExit, (U32)pulNotificationValue, xTicksToWait)
+#define traceTASK_NOTIFY( uxIndexToNotify )                           SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFY, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue)
+#define traceTASK_NOTIFY_FROM_ISR( uxIndexToNotify )                  SYSVIEW_RecordU32x5(apiFastID_OFFSET + apiID_XTASKGENERICNOTIFYFROMISR, SEGGER_SYSVIEW_ShrinkId((U32)pxTCB), ulValue, eAction, (U32)pulPreviousNotificationValue, (U32)pxHigherPriorityTaskWoken)
+#define traceTASK_NOTIFY_WAIT( uxIndexToWait )                        SYSVIEW_RecordU32x4(apiFastID_OFFSET + apiID_XTASKNOTIFYWAIT, ulBitsToClearOnEntry, ulBitsToClearOnExit, (U32)pulNotificationValue, xTicksToWait)
 
 #define traceQUEUE_CREATE( pxNewQueue )                               SEGGER_SYSVIEW_RecordU32x3(apiFastID_OFFSET + apiID_XQUEUEGENERICCREATE, uxQueueLength, uxItemSize, ucQueueType)
 #define traceQUEUE_DELETE( pxQueue )                                  SEGGER_SYSVIEW_RecordU32(apiFastID_OFFSET + apiID_VQUEUEDELETE, SEGGER_SYSVIEW_ShrinkId((U32)pxQueue))
@@ -290,12 +290,12 @@ Notes:
   #define traceTASK_SWITCHED_IN()                   if(prvGetTCBFromHandle(NULL) == xTaskGetIdleTaskHandle()) {           \
                                                       SEGGER_SYSVIEW_OnIdle();                                          \
                                                     } else {                                                            \
-                                                      SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[xPortGetCoreID()]); \
+                                                      SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[cpu_hal_get_core_id()]); \
                                                     }
 #else
   #define traceTASK_SWITCHED_IN()                   {                                                                   \
-                                                      if (memcmp(pxCurrentTCB[xPortGetCoreID()]->pcTaskName, "IDLE", 5) != 0) { \
-                                                        SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[xPortGetCoreID()]);    \
+                                                      if (memcmp(pxCurrentTCB[cpu_hal_get_core_id()]->pcTaskName, "IDLE", 5) != 0) { \
+                                                        SEGGER_SYSVIEW_OnTaskStartExec((U32)pxCurrentTCB[cpu_hal_get_core_id()]);    \
                                                       } else {                                                          \
                                                         SEGGER_SYSVIEW_OnIdle();                                        \
                                                       }                                                                 \
@@ -305,14 +305,15 @@ Notes:
 #define traceMOVED_TASK_TO_READY_STATE(pxTCB)       SEGGER_SYSVIEW_OnTaskStartReady((U32)pxTCB)
 #define traceREADDED_TASK_TO_READY_STATE(pxTCB)
 
-#define traceMOVED_TASK_TO_DELAYED_LIST()           SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[xPortGetCoreID()],  (1u << 2))
-#define traceMOVED_TASK_TO_OVERFLOW_DELAYED_LIST()  SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[xPortGetCoreID()],  (1u << 2))
+#define traceMOVED_TASK_TO_DELAYED_LIST()           SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[cpu_hal_get_core_id()],  (1u << 2))
+#define traceMOVED_TASK_TO_OVERFLOW_DELAYED_LIST()  SEGGER_SYSVIEW_OnTaskStopReady((U32)pxCurrentTCB[cpu_hal_get_core_id()],  (1u << 2))
 #define traceMOVED_TASK_TO_SUSPENDED_LIST(pxTCB)    SEGGER_SYSVIEW_OnTaskStopReady((U32)pxTCB,         ((3u << 3) | 3))
 
 #define traceISR_EXIT_TO_SCHEDULER()                SEGGER_SYSVIEW_RecordExitISRToScheduler()
 #define traceISR_EXIT()                             SEGGER_SYSVIEW_RecordExitISR()
 #define traceISR_ENTER(_n_)                         SEGGER_SYSVIEW_RecordEnterISR(_n_)
 
+
 /*********************************************************************
 *
 *       API functions

components/app_trace/sys_view/esp32/SEGGER_RTT_esp32.c → components/app_trace/sys_view/esp/SEGGER_RTT_esp.c

@@ -20,10 +20,10 @@ const static char *TAG = "segger_rtt";
 // size of down channel data buf
 #define SYSVIEW_DOWN_BUF_SIZE   32
 #define SEGGER_STOP_WAIT_TMO    1000000 //us
-#if CONFIG_SYSVIEW_BUF_WAIT_TMO == -1
+#if CONFIG_APPTRACE_SV_BUF_WAIT_TMO == -1
 #define SEGGER_HOST_WAIT_TMO    ESP_APPTRACE_TMO_INFINITE
 #else
-#define SEGGER_HOST_WAIT_TMO    CONFIG_SYSVIEW_BUF_WAIT_TMO
+#define SEGGER_HOST_WAIT_TMO    CONFIG_APPTRACE_SV_BUF_WAIT_TMO
 #endif
 
 static uint8_t s_events_buf[SYSVIEW_EVENTS_BUF_SZ];
@@ -39,7 +39,7 @@ static uint8_t s_down_buf[SYSVIEW_DOWN_BUF_SIZE];
 
 /*********************************************************************
 *
-*       SEGGER_RTT_ESP32_FlushNoLock()
+*       SEGGER_RTT_ESP_FlushNoLock()
 *
 *  Function description
 *    Flushes buffered events.
@@ -51,7 +51,7 @@ static uint8_t s_down_buf[SYSVIEW_DOWN_BUF_SIZE];
 *  Return value
 *    None.
 */
-void SEGGER_RTT_ESP32_FlushNoLock(unsigned long min_sz, unsigned long tmo)
+void SEGGER_RTT_ESP_FlushNoLock(unsigned long min_sz, unsigned long tmo)
 {
     esp_err_t res;
     if (s_events_buf_filled > 0) {
@@ -70,7 +70,7 @@ void SEGGER_RTT_ESP32_FlushNoLock(unsigned long min_sz, unsigned long tmo)
 
 /*********************************************************************
 *
-*       SEGGER_RTT_ESP32_Flush()
+*       SEGGER_RTT_ESP_Flush()
 *
 *  Function description
 *    Flushes buffered events.
@@ -82,10 +82,10 @@ void SEGGER_RTT_ESP32_FlushNoLock(unsigned long min_sz, unsigned long tmo)
 *  Return value
 *    None.
 */
-void SEGGER_RTT_ESP32_Flush(unsigned long min_sz, unsigned long tmo)
+void SEGGER_RTT_ESP_Flush(unsigned long min_sz, unsigned long tmo)
 {
     SEGGER_SYSVIEW_LOCK();
-    SEGGER_RTT_ESP32_FlushNoLock(min_sz, tmo);
+    SEGGER_RTT_ESP_FlushNoLock(min_sz, tmo);
     SEGGER_SYSVIEW_UNLOCK();
 }
 
@@ -147,7 +147,7 @@ unsigned SEGGER_RTT_WriteSkipNoLock(unsigned BufferIndex, const void* pBuffer, u
       ESP_LOGE(TAG, "Too large event %u bytes!", NumBytes);
       return 0;
   }
-  if (xPortGetCoreID()) { // dual core specific code
+  if (cpu_hal_get_core_id()) { // dual core specific code
     // use the highest - 1 bit of event ID to indicate core ID
     // the highest bit can not be used due to event ID encoding method
     // this reduces supported ID range to [0..63] (for 1 byte IDs) plus [128..16383] (for 2 bytes IDs)
@@ -167,7 +167,7 @@ unsigned SEGGER_RTT_WriteSkipNoLock(unsigned BufferIndex, const void* pBuffer, u
   memcpy(&s_events_buf[s_events_buf_filled], pBuffer, NumBytes);
   s_events_buf_filled += NumBytes;
   if (event_id == SYSVIEW_EVTID_TRACE_STOP) {
-    SEGGER_RTT_ESP32_FlushNoLock(0, SEGGER_STOP_WAIT_TMO);
+    SEGGER_RTT_ESP_FlushNoLock(0, SEGGER_STOP_WAIT_TMO);
   }
   return NumBytes;
 }

components/app_trace/sys_view/ext/heap_trace_module.c

@@ -52,7 +52,7 @@ esp_err_t esp_sysview_heap_trace_start(uint32_t tmo)
 esp_err_t esp_sysview_heap_trace_stop(void)
 {
     ESP_EARLY_LOGV(TAG, "%s", __func__);
-    SEGGER_RTT_ESP32_Flush(0, ESP_APPTRACE_TMO_INFINITE);
+    SEGGER_RTT_ESP_Flush(0, ESP_APPTRACE_TMO_INFINITE);
     return ESP_OK;
 }
 

components/app_trace/test/CMakeLists.txt

@@ -1,3 +1,3 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock)
+                    PRIV_REQUIRES cmock driver)

components/app_trace/test/test_trace.c

@@ -88,7 +88,7 @@ static void esp_apptrace_test_timer_init(int timer_group, int timer_idx, uint32_
     timer_enable_intr(timer_group, timer_idx);
 }
 
-#if CONFIG_SYSVIEW_ENABLE == 0
+#if CONFIG_APPTRACE_SV_ENABLE == 0
 #define ESP_APPTRACE_TEST_WRITE(_b_, _s_)            esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, ESP_APPTRACE_TMO_INFINITE)
 #define ESP_APPTRACE_TEST_WRITE_FROM_ISR(_b_, _s_)   esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, 0UL)
 #define ESP_APPTRACE_TEST_WRITE_NOWAIT(_b_, _s_)     esp_apptrace_write(ESP_APPTRACE_DEST_TRAX, _b_, _s_, 0)
@@ -214,7 +214,7 @@ static void esp_apptrace_dummy_task(void *p)
 
     i = 0;
     while (!arg->stop) {
-        ESP_APPTRACE_TEST_LOGD("%x: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), xPortGetCoreID(), i++);
+        ESP_APPTRACE_TEST_LOGD("%x: dummy task work %d.%d", xTaskGetCurrentTaskHandle(), cpu_hal_get_core_id(), i++);
         if (tmo_ticks) {
             vTaskDelay(tmo_ticks);
         }
@@ -259,7 +259,7 @@ static void esp_apptrace_test_task(void *p)
                 ESP_APPTRACE_TEST_LOGE("Failed to timer_isr_register (%d)!", res);
                 goto on_fail;
             }
-            *(uint32_t *)arg->timers[i].data.buf = ((uint32_t)inth[i]) | (1 << 31) | (xPortGetCoreID() ? 0x1 : 0);
+            *(uint32_t *)arg->timers[i].data.buf = ((uint32_t)inth[i]) | (1 << 31) | (cpu_hal_get_core_id() ? 0x1 : 0);
             ESP_APPTRACE_TEST_LOGI("%x: start timer %x period %u us", xTaskGetCurrentTaskHandle(), inth[i], arg->timers[i].data.period);
             res = timer_start(arg->timers[i].group, arg->timers[i].id);
             if (res != ESP_OK) {
@@ -269,7 +269,7 @@ static void esp_apptrace_test_task(void *p)
         }
     }
 
-    *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle() | (xPortGetCoreID() ? 0x1 : 0);
+    *(uint32_t *)arg->data.buf = (uint32_t)xTaskGetCurrentTaskHandle() | (cpu_hal_get_core_id() ? 0x1 : 0);
     arg->data.wr_cnt = 0;
     arg->data.wr_err = 0;
     while (!arg->stop) {
@@ -749,7 +749,7 @@ static void esp_logtrace_task(void *p)
         ESP_LOGI(TAG, "%p: sample print 4 %c", xTaskGetCurrentTaskHandle(), ((i & 0xFF) % 95) + 32);
         ESP_LOGI(TAG, "%p: sample print 5 %f", xTaskGetCurrentTaskHandle(), 1.0);
         ESP_LOGI(TAG, "%p: sample print 6 %f", xTaskGetCurrentTaskHandle(), 3.45);
-        ESP_LOGI(TAG, "%p: logtrace task work %d.%d", xTaskGetCurrentTaskHandle(), xPortGetCoreID(), i);
+        ESP_LOGI(TAG, "%p: logtrace task work %d.%d", xTaskGetCurrentTaskHandle(), cpu_hal_get_core_id(), i);
         if (++i == 10000) {
             break;
         }

components/app_update/CMakeLists.txt

@@ -2,7 +2,7 @@ idf_component_register(SRCS "esp_ota_ops.c"
                             "esp_app_desc.c"
                     INCLUDE_DIRS "include"
                     REQUIRES spi_flash partition_table bootloader_support
-                    PRIV_REQUIRES esptool_py)
+                    PRIV_REQUIRES esptool_py efuse)
 
 # esp_app_desc structure is added as an undefined symbol because otherwise the
 # linker will ignore this structure as it has no other files depending on it.
@@ -46,6 +46,7 @@ if(NOT BOOTLOADER_BUILD)
 
         add_custom_target(blank_ota_data ALL DEPENDS ${blank_otadata_file})
         add_dependencies(flash blank_ota_data)
+        add_dependencies(encrypted-flash blank_ota_data)
 
         set(otatool_py ${python} ${COMPONENT_DIR}/otatool.py)
 

components/app_update/esp_app_desc.c

@@ -1,16 +1,8 @@
-// Copyright 2017-2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #include <assert.h>
 #include <sys/param.h>

components/app_update/esp_ota_ops.c

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #include <stdint.h>
 #include <stdbool.h>
@@ -19,8 +11,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <assert.h>
-#include <freertos/FreeRTOS.h>
-#include <freertos/task.h>
 
 #include "esp_err.h"
 #include "esp_partition.h"
@@ -381,7 +371,7 @@ static esp_err_t rewrite_ota_seq(esp_ota_select_entry_t *two_otadata, uint32_t s
     }
 }
 
-static uint8_t get_ota_partition_count(void)
+uint8_t esp_ota_get_app_partition_count(void)
 {
     uint16_t ota_app_count = 0;
     while (esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_OTA_MIN + ota_app_count, NULL) != NULL) {
@@ -399,7 +389,7 @@ static esp_err_t esp_rewrite_ota_data(esp_partition_subtype_t subtype)
         return ESP_ERR_NOT_FOUND;
     }
 
-    uint8_t ota_app_count = get_ota_partition_count();
+    uint8_t ota_app_count = esp_ota_get_app_partition_count();
     if (SUB_TYPE_ID(subtype) >= ota_app_count) {
         return ESP_ERR_INVALID_ARG;
     }
@@ -517,7 +507,7 @@ const esp_partition_t *esp_ota_get_boot_partition(void)
         return NULL;
     }
 
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     ESP_LOGD(TAG, "found ota app max = %d", ota_app_count);
 
     if ((bootloader_common_ota_select_invalid(&otadata[0]) &&
@@ -662,7 +652,7 @@ bool esp_ota_check_rollback_is_possible(void)
         return false;
     }
 
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (ota_app_count == 0) {
         return false;
     }
@@ -720,7 +710,7 @@ static esp_err_t esp_ota_current_ota_is_workable(bool valid)
     }
 
     int active_otadata = bootloader_common_get_active_otadata(otadata);
-    if (active_otadata != -1 && get_ota_partition_count() != 0) {
+    if (active_otadata != -1 && esp_ota_get_app_partition_count() != 0) {
         if (valid == true && otadata[active_otadata].ota_state != ESP_OTA_IMG_VALID) {
             otadata[active_otadata].ota_state = ESP_OTA_IMG_VALID;
             ESP_LOGD(TAG, "OTA[current] partition is marked as VALID");
@@ -789,7 +779,7 @@ const esp_partition_t* esp_ota_get_last_invalid_partition(void)
 
     int invalid_otadata = get_last_invalid_otadata(otadata);
 
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (invalid_otadata != -1 && ota_app_count != 0) {
         int ota_slot = (otadata[invalid_otadata].ota_seq - 1) % ota_app_count;
         ESP_LOGD(TAG, "Find invalid ota_%d app", ESP_PARTITION_SUBTYPE_APP_OTA_MIN + ota_slot);
@@ -817,7 +807,7 @@ esp_err_t esp_ota_get_state_partition(const esp_partition_t *partition, esp_ota_
     }
 
     esp_ota_select_entry_t otadata[2];
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (read_otadata(otadata) == NULL || ota_app_count == 0) {
         return ESP_ERR_NOT_FOUND;
     }
@@ -849,7 +839,7 @@ esp_err_t esp_ota_erase_last_boot_app_partition(void)
     }
 
     int active_otadata = bootloader_common_get_active_otadata(otadata);
-    int ota_app_count = get_ota_partition_count();
+    int ota_app_count = esp_ota_get_app_partition_count();
     if (active_otadata == -1 || ota_app_count == 0) {
         return ESP_FAIL;
     }
@@ -901,8 +891,71 @@ esp_err_t esp_ota_revoke_secure_boot_public_key(esp_ota_secure_boot_public_key_i
         return ESP_ERR_INVALID_ARG;
     }
 
-    ets_secure_boot_revoke_public_key_digest(index);
-    ESP_LOGI(TAG, "Revoked signature block %d.", index);
+    esp_image_sig_public_key_digests_t app_digests = { 0 };
+    esp_err_t err = esp_secure_boot_get_signature_blocks_for_running_app(true, &app_digests);
+    if (err != ESP_OK || app_digests.num_digests == 0) {
+        ESP_LOGE(TAG, "This app is not signed, but check signature on update is enabled in config. It won't be possible to verify any update.");
+        return ESP_FAIL;
+    }
+
+    esp_err_t ret;
+    ets_secure_boot_key_digests_t trusted_keys;
+    ret = esp_secure_boot_read_key_digests(&trusted_keys);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "Could not read the secure boot key digests from efuse. Aborting..");
+        return ESP_FAIL;
+    }
+
+    if (trusted_keys.key_digests[index] == NULL) {
+        ESP_LOGI(TAG, "Trusted Key block(%d) already revoked.", index);
+        return ESP_OK;
+    }
+
+    esp_image_sig_public_key_digests_t trusted_digests = { 0 };
+    for (unsigned i = 0; i < SECURE_BOOT_NUM_BLOCKS; i++) {
+        if (i == index) {
+            continue; // omitting - to find if there is a valid key after revoking this digest
+        }
+
+        if (trusted_keys.key_digests[i] != NULL) {
+            bool all_zeroes = true;
+            for (unsigned j = 0; j < ESP_SECURE_BOOT_DIGEST_LEN; j++) {
+                all_zeroes = all_zeroes && (*(uint8_t *)(trusted_keys.key_digests[i] + j) == 0);
+            }
+            if (!all_zeroes) {
+                memcpy(trusted_digests.key_digests[trusted_digests.num_digests++], (uint8_t *)trusted_keys.key_digests[i], ESP_SECURE_BOOT_DIGEST_LEN);
+            } else {
+                ESP_LOGD(TAG, "Empty trusted key block (%d).", i);
+            }
+        }
+    }
+
+    bool match = false;
+    for (unsigned i = 0; i < trusted_digests.num_digests; i++) {
+        if (match == true) {
+            break;
+        }
+
+        for (unsigned j = 0; j < app_digests.num_digests; j++) {
+            if (memcmp(trusted_digests.key_digests[i], app_digests.key_digests[j], ESP_SECURE_BOOT_DIGEST_LEN) == 0) {
+                ESP_LOGI(TAG, "App key block(%d) matches Trusted key block(%d)[%d -> Next active trusted key block].", j, i, i);
+                esp_err_t err = esp_efuse_set_digest_revoke(index);
+                if (err != ESP_OK) {
+                    ESP_LOGE(TAG, "Failed to revoke digest (0x%x).", err);
+                    return ESP_FAIL;
+                }
+                ESP_LOGI(TAG, "Revoked signature block %d.", index);
+                match = true;
+                break;
+            }
+        }
+    }
+
+    if (match == false) {
+        ESP_LOGE(TAG, "Running app doesn't have another valid secure boot key. Cannot revoke current key(%d).", index);
+        return ESP_FAIL;
+    }
+
     return ESP_OK;
 }
 #endif

components/app_update/include/esp_ota_ops.h

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #ifndef _OTA_OPS_H
 #define _OTA_OPS_H
@@ -22,6 +14,7 @@
 #include "esp_partition.h"
 #include "esp_image_format.h"
 #include "esp_flash_partitions.h"
+#include "soc/soc_caps.h"
 
 #ifdef __cplusplus
 extern "C"
@@ -251,6 +244,14 @@ const esp_partition_t* esp_ota_get_next_update_partition(const esp_partition_t *
  */
 esp_err_t esp_ota_get_partition_description(const esp_partition_t *partition, esp_app_desc_t *app_desc);
 
+/**
+ * @brief Returns number of ota partitions provided in partition table.
+ *
+ * @return
+ *  - Number of OTA partitions
+ */
+uint8_t esp_ota_get_app_partition_count(void);
+
 /**
  * @brief This function is called to indicate that the running app is working well.
  *

components/app_update/otatool.py

@@ -3,19 +3,8 @@
 # otatool is used to perform ota-level operations - flashing ota partition
 # erasing ota partition and switching ota partition
 #
-# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 from __future__ import division, print_function
 
 import argparse
@@ -245,10 +234,6 @@ def _erase_ota_partition(target, ota_id):
 
 
 def main():
-    if sys.version_info[0] < 3:
-        print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     global quiet
 
     parser = argparse.ArgumentParser('ESP-IDF OTA Partitions Tool')

components/app_update/test/test_app_desc.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <string.h>
 #include "esp_ota_ops.h"
 #include "unity.h"

components/app_update/test/test_ota_ops.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

components/app_update/test/test_switch_ota.c

@@ -1,3 +1,8 @@
+/*
+ * SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 /*
  * Tests for switching between partitions: factory, OTAx, test.
  */
@@ -9,10 +14,8 @@
 
 #if CONFIG_IDF_TARGET_ESP32
 #include "esp32/rom/spi_flash.h"
-#include "esp32/rom/rtc.h"
 #elif CONFIG_IDF_TARGET_ESP32S2
 #include "esp32s2/rom/spi_flash.h"
-#include "esp32s2/rom/rtc.h"
 #endif
 
 #include "freertos/FreeRTOS.h"

components/asio/CMakeLists.txt

@@ -1,3 +1,11 @@
+if(NOT CONFIG_LWIP_IPV6 AND NOT CMAKE_BUILD_EARLY_EXPANSION)
+    # note: the component is still included in the build so it can become visible again in config
+    # without needing to re-run CMake. However no source or header files are built.
+    message(STATUS "IPV6 support is disabled so the asio component will not be built")
+    idf_component_register()
+    return()
+endif()
+
 set(asio_sources "asio/asio/src/asio.cpp")
 
 if(CONFIG_ASIO_SSL_SUPPORT)

components/asio/Kconfig

@@ -1,4 +1,6 @@
 menu "ESP-ASIO"
+    visible if LWIP_IPV6
+
     config ASIO_SSL_SUPPORT
         bool "Enable SSL/TLS support of ASIO"
         default n

components/asio/component.mk

@@ -1,3 +1,5 @@
+ifdef CONFIG_LWIP_IPV6
+
 COMPONENT_ADD_INCLUDEDIRS := asio/asio/include port/include
 COMPONENT_PRIV_INCLUDEDIRS := private_include
 COMPONENT_SRCDIRS := asio/asio/src port/src
@@ -7,3 +9,5 @@ COMPONENT_OBJEXCLUDE := asio/asio/src/asio_ssl.o port/src/esp_asio_openssl_stubs
 endif
 
 COMPONENT_SUBMODULES += asio
+
+endif # CONFIG_LWIP_IPV6

components/asio/port/include/esp_asio_config.h

@@ -1,16 +1,8 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #ifndef _ESP_ASIO_CONFIG_H_
 #define _ESP_ASIO_CONFIG_H_
 

components/asio/port/include/esp_exception.h

@@ -1,17 +1,9 @@
 
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #ifndef _ESP_EXCEPTION_H_
 #define _ESP_EXCEPTION_H_
 

components/asio/port/include/openssl/conf.h

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #ifndef _ESP_ASIO_OPENSSL_CONF_H
 #define _ESP_ASIO_OPENSSL_CONF_H

components/asio/port/include/openssl/dh.h

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #ifndef _ESP_ASIO_OPENSSL_DH_STUB_H
 #define _ESP_ASIO_OPENSSL_DH_STUB_H

components/asio/port/include/openssl/esp_asio_openssl_stubs.h

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #ifndef _ESP_ASIO_OPENSSL_STUBS_H
 #define _ESP_ASIO_OPENSSL_STUBS_H

components/asio/port/include/openssl/rsa.h

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #ifndef _ESP_ASIO_OPENSSL_RSA_STUB_H
 #define _ESP_ASIO_OPENSSL_RSA_STUB_H

components/asio/port/include/openssl/x509v3.h

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #ifndef _ESP_ASIO_OPENSSL_X509V3_STUB_H
 #define _ESP_ASIO_OPENSSL_X509V3_STUB_H

components/asio/port/src/esp_asio_openssl_stubs.c

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 //
 
 #include "esp_asio_config.h"

components/bootloader/Kconfig.projbuild

@@ -117,8 +117,8 @@ menu "Bootloader config"
             Allows to reset the device to factory settings:
             - clear one or more data partitions;
             - boot from "factory" partition.
-            The factory reset will occur if there is a GPIO input pulled low while device starts up.
-            See settings below.
+            The factory reset will occur if there is a GPIO input held at the configured level while
+            device starts up. See settings below.
 
     config BOOTLOADER_NUM_PIN_FACTORY_RESET
         int "Number of the GPIO input for factory reset"
@@ -127,9 +127,23 @@ menu "Bootloader config"
         range 0 44 if IDF_TARGET_ESP32S2
         default 4
         help
-            The selected GPIO will be configured as an input with internal pull-up enabled.
-            To trigger a factory reset, this GPIO must be pulled low on reset.
-            Note that GPIO34-39 do not have an internal pullup and an external one must be provided.
+            The selected GPIO will be configured as an input with internal pull-up enabled (note that on some SoCs.
+            not all pins have an internal pull-up, consult the hardware datasheet for details.) To trigger a factory
+            reset, this GPIO must be held high or low (as configured) on startup.
+
+    choice BOOTLOADER_FACTORY_RESET_PIN_LEVEL
+        bool "Factory reset GPIO level"
+        depends on BOOTLOADER_FACTORY_RESET
+        default BOOTLOADER_FACTORY_RESET_PIN_LOW
+        help
+            Pin level for factory reset, can be triggered on low or high.
+
+        config BOOTLOADER_FACTORY_RESET_PIN_LOW
+            bool "Reset on GPIO low"
+
+        config BOOTLOADER_FACTORY_RESET_PIN_HIGH
+            bool "Reset on GPIO high"
+    endchoice
 
     config BOOTLOADER_OTA_DATA_ERASE
         bool "Clear OTA data on factory reset (select factory partition)"
@@ -197,8 +211,11 @@ menu "Bootloader config"
         depends on BOOTLOADER_WDT_ENABLE
         default n
         help
-            If it is set, the client must itself reset or disable rtc_wdt in their code (app_main()).
-            Otherwise rtc_wdt will be disabled before calling app_main function.
+            If this option is set, the ESP-IDF app must explicitly reset, feed, or disable the rtc_wdt in
+            the app's own code.
+            If this option is not set (default), then rtc_wdt will be disabled by ESP-IDF before calling
+            the app_main() function.
+
             Use function rtc_wdt_feed() for resetting counter of rtc_wdt.
             Use function rtc_wdt_disable() for disabling rtc_wdt.
 
@@ -267,10 +284,14 @@ menu "Bootloader config"
         bool "Emulate operations with efuse secure version(only test)"
         default n
         depends on BOOTLOADER_APP_ANTI_ROLLBACK
+        select EFUSE_VIRTUAL
+        select EFUSE_VIRTUAL_KEEP_IN_FLASH
         help
-            This option allow emulate read/write operations with efuse secure version.
-            It allow to test anti-rollback implemention without permanent write eFuse bits.
-            In partition table should be exist this partition `emul_efuse, data, 5, , 0x2000`.
+            This option allows to emulate read/write operations with all eFuses and efuse secure version.
+            It allows to test anti-rollback implemention without permanent write eFuse bits.
+            There should be an entry in partition table with following details: `emul_efuse, data, efuse, , 0x2000`.
+
+            This option enables: EFUSE_VIRTUAL and EFUSE_VIRTUAL_KEEP_IN_FLASH.
 
     config BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP
         bool "Skip image validation when exiting deep sleep"
@@ -365,6 +386,15 @@ menu "Bootloader config"
             in this area of memory, you can increase it. It must be a multiple of 4 bytes.
             This area (rtc_retain_mem_t) is reserved and has access from the bootloader and an application.
 
+    config BOOTLOADER_FLASH_XMC_SUPPORT
+        bool "Enable the support for flash chips of XMC (READ HELP FIRST)"
+        default y
+        help
+            Perform the startup flow recommended by XMC. Please consult XMC for the details of this flow.
+            XMC chips will be forbidden to be used, when this option is disabled.
+
+            DON'T DISABLE THIS UNLESS YOU KNOW WHAT YOU ARE DOING.
+
 endmenu  # Bootloader
 
 
@@ -394,12 +424,12 @@ menu "Security features"
     config SECURE_BOOT_SUPPORTS_RSA
         bool
         default y
-        depends on ESP32_REV_MIN_3 || IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32C3
+        depends on ESP32_REV_MIN_3 || IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32S3
 
     config SECURE_TARGET_HAS_SECURE_ROM_DL_MODE
         bool
         default y
-        depends on IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32C3
+        depends on IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32S3
 
 
     config SECURE_SIGNED_APPS_NO_SECURE_BOOT
@@ -471,7 +501,8 @@ menu "Security features"
     config SECURE_BOOT
         bool "Enable hardware Secure Boot in bootloader (READ DOCS FIRST)"
         default n
-        depends on IDF_TARGET_ESP32 || IDF_TARGET_ESP32S2 || ESP32C3_REV_MIN_3
+        depends on IDF_TARGET_ESP32 || IDF_TARGET_ESP32S2 || ESP32C3_REV_MIN_3 || IDF_TARGET_ESP32S3
+        select ESPTOOLPY_NO_STUB if !IDF_TARGET_ESP32 && !IDF_TARGET_ESP32S2
         help
             Build a bootloader which enables Secure Boot on first boot.
 
@@ -634,7 +665,7 @@ menu "Security features"
     choice SECURE_FLASH_ENCRYPTION_KEYSIZE
         bool "Size of generated AES-XTS key"
         default SECURE_FLASH_ENCRYPTION_AES128
-        depends on IDF_TARGET_ESP32S2 && SECURE_FLASH_ENC_ENABLED
+        depends on (IDF_TARGET_ESP32S2 || IDF_TARGET_ESP32S3) && SECURE_FLASH_ENC_ENABLED
         help
             Size of generated AES-XTS key.
 
@@ -672,6 +703,8 @@ menu "Security features"
 
         config SECURE_FLASH_ENCRYPTION_MODE_RELEASE
             bool "Release"
+            select PARTITION_TABLE_MD5 if !ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS
+
     endchoice
 
     menu "Potentially insecure options"
@@ -732,9 +765,17 @@ menu "Security features"
                 efuse when Secure Boot is enabled. This prevents any more efuses from being read protected.
 
                 If this option is set, it will remain possible to write the EFUSE_RD_DIS efuse field after Secure
-                Boot is enabled. This may allow an attacker to read-protect the BLK2 efuse holding the public
-                key digest, causing an immediate denial of service and possibly allowing an additional fault
-                injection attack to bypass the signature protection.
+                Boot is enabled. This may allow an attacker to read-protect the BLK2 efuse (for ESP32) and
+                BLOCK4-BLOCK10 (i.e. BLOCK_KEY0-BLOCK_KEY5)(for other chips) holding the public key digest, causing an
+                immediate denial of service and possibly allowing an additional fault injection attack to
+                bypass the signature protection.
+
+                NOTE: Once a BLOCK is read-protected, the application will read all zeros from that block
+
+                NOTE: If "UART ROM download mode (Permanently disabled (recommended))" or
+                "UART ROM download mode (Permanently switch to Secure mode (recommended))" is set,
+                then it is __NOT__ possible to read/write efuses using espefuse.py utility.
+                However, efuse can be read/written from the application
 
         config SECURE_BOOT_ALLOW_UNUSED_DIGEST_SLOTS
             bool "Leave unused digest slots available (not revoke)"
@@ -852,6 +893,8 @@ menu "Security features"
                 It is also possible to enable secure download mode at runtime by calling
                 esp_efuse_enable_rom_secure_download_mode()
 
+                Note: Secure Download mode is not available for ESP32 (includes revisions till ECO3).
+
         config SECURE_INSECURE_ALLOW_DL_MODE
             bool "UART ROM download mode (Enabled (not recommended))"
             help

components/bootloader/Makefile.projbuild

@@ -27,6 +27,9 @@ BOOTLOADER_OFFSET := 0x1000
 # NB: Some variables are cleared in the environment, not
 # overriden, because they need to be re-defined in the child
 # project.
+#
+# Pass PROJECT_PATH variable, it will let the subproject look
+# for user defined bootloader component(s).
 BOOTLOADER_MAKE= +\
 	PROJECT_PATH= \
 	COMPONENT_DIRS= \
@@ -35,7 +38,8 @@ BOOTLOADER_MAKE= +\
 	BUILD_DIR_BASE=$(BOOTLOADER_BUILD_DIR) \
 	TEST_COMPONENTS= \
 	TESTS_ALL= \
-	EXCLUDE_COMPONENTS=
+	EXCLUDE_COMPONENTS= \
+	PROJECT_SOURCE_DIR=$(PROJECT_PATH)
 
 .PHONY: bootloader-clean bootloader-flash bootloader-list-components bootloader $(BOOTLOADER_BIN)
 

components/bootloader/project_include.cmake

@@ -117,6 +117,7 @@ externalproject_add(bootloader
     CMAKE_ARGS  -DSDKCONFIG=${sdkconfig} -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target}
                 -DPYTHON_DEPS_CHECKED=1 -DPYTHON=${python}
                 -DEXTRA_COMPONENT_DIRS=${CMAKE_CURRENT_LIST_DIR}
+                -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
                 ${sign_key_arg} ${ver_key_arg}
                 # LEGACY_INCLUDE_COMMON_HEADERS has to be passed in via cache variable since
                 # the bootloader common component requirements depends on this and

components/bootloader/subproject/CMakeLists.txt

@@ -33,6 +33,21 @@ set(COMPONENTS
     efuse
     esp_system
     newlib)
+
+# Make EXTRA_COMPONENT_DIRS variable to point to the bootloader_components directory
+# of the project being compiled
+set(PROJECT_EXTRA_COMPONENTS "${PROJECT_SOURCE_DIR}/bootloader_components")
+set(EXTRA_COMPONENT_DIRS "${PROJECT_EXTRA_COMPONENTS}" APPEND)
+
+# Consider each directory in project's bootloader_components as a component to be compiled
+file(GLOB proj_components RELATIVE ${PROJECT_EXTRA_COMPONENTS} ${PROJECT_EXTRA_COMPONENTS}/*)
+foreach(component ${proj_components})
+  # Only directories are considered as components
+  if(IS_DIRECTORY ${curdir}/${child})
+    list(APPEND COMPONENTS ${component})
+  endif()
+endforeach()
+
 set(BOOTLOADER_BUILD 1)
 include("${IDF_PATH}/tools/cmake/project.cmake")
 set(common_req log esp_rom esp_common esp_hw_support hal newlib)

components/bootloader/subproject/Makefile

@@ -14,9 +14,10 @@ COMPONENTS := esp_hw_support esptool_py bootloader_support log spi_flash micro-e
 CFLAGS =
 CXXFLAGS =
 
-#We cannot include the idf_target, esp_common component directly but we need their includes.
+#We cannot include the some components like idf_target, esp_common directly but we need their includes.
 CFLAGS += -I $(IDF_PATH)/components/$(IDF_TARGET)/include
 CFLAGS += -I $(IDF_PATH)/components/esp_common/include
+CFLAGS += -I $(IDF_PATH)/components/newlib/platform_include
 CFLAGS += -I $(IDF_PATH)/components/xtensa/include -I $(IDF_PATH)/components/xtensa/$(IDF_TARGET)/include
 
 # The bootloader pseudo-component is also included in this build, for its Kconfig.projbuild to be included.

components/bootloader/subproject/components/micro-ecc/uECC_verify_antifault.c

@@ -1,8 +1,10 @@
-/* Copyright 2014, Kenneth MacKay. Licensed under the BSD 2-clause license.
-
-   Modifications Copyright 2020, Espressif Systems (Shanghai) PTE LTD. Licensed under the BSD
-   2-clause license.
-*/
+/*
+ * SPDX-FileCopyrightText: 2014, Kenneth MacKay
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * SPDX-FileContributor: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ */
 
 /* uECC_verify() calls a number of static functions form here and
    uses other definitions, so we just build that whole source file here and then append

components/bootloader/subproject/components/micro-ecc/uECC_verify_antifault.h

@@ -1,8 +1,10 @@
-/* Copyright 2014, Kenneth MacKay. Licensed under the BSD 2-clause license.
-
-   Modifications Copyright 2020, Espressif Systems (Shanghai) PTE LTD. Licensed under the BSD
-   2-clause license.
-*/
+/*
+ * SPDX-FileCopyrightText: 2014, Kenneth MacKay
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * SPDX-FileContributor: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ */
 #pragma once
 #include "uECC.h"
 

components/bootloader/subproject/main/CMakeLists.txt

@@ -4,8 +4,10 @@ idf_component_register(SRCS "bootloader_start.c"
 idf_build_get_property(target IDF_TARGET)
 set(scripts "ld/${target}/bootloader.ld")
 
-if(NOT CONFIG_IDF_TARGET_ESP32C3)
+if(NOT CONFIG_IDF_TARGET_ESP32H2)
     list(APPEND scripts "ld/${target}/bootloader.rom.ld")
 endif()
 
 target_linker_script(${COMPONENT_LIB} INTERFACE "${scripts}")
+
+target_link_libraries(${COMPONENT_LIB} INTERFACE "-u bootloader_hooks_include")

components/bootloader/subproject/main/bootloader_hooks.h

@@ -0,0 +1,38 @@
+// Copyright 2015-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef BOOTLOADER_HOOKS_H
+#define BOOTLOADER_HOOKS_H
+
+/**
+ * @file The 2nd stage bootloader can be overriden or completed by an application.
+ * The functions declared here are weak, and thus, are meant to be defined by a user
+ * project, if required.
+ * Please check `custom_bootloader` ESP-IDF examples for more details about this feature.
+ */
+
+
+/**
+ * @brief Function executed *before* the second stage bootloader initialization,
+ * if provided.
+ */
+void __attribute__((weak)) bootloader_before_init(void);
+
+/**
+ * @brief Function executed *after* the second stage bootloader initialization,
+ * if provided.
+ */
+void __attribute__((weak)) bootloader_after_init(void);
+
+#endif // BOOTLOADER_HOOKS_H

components/bootloader/subproject/main/bootloader_start.c

@@ -16,6 +16,7 @@
 #include "bootloader_init.h"
 #include "bootloader_utility.h"
 #include "bootloader_common.h"
+#include "bootloader_hooks.h"
 
 static const char *TAG = "boot";
 
@@ -29,11 +30,21 @@ static int selected_boot_partition(const bootloader_state_t *bs);
  */
 void __attribute__((noreturn)) call_start_cpu0(void)
 {
+    // (0. Call the before-init hook, if available)
+    if (bootloader_before_init) {
+        bootloader_before_init();
+    }
+
     // 1. Hardware initialization
     if (bootloader_init() != ESP_OK) {
         bootloader_reset();
     }
 
+    // (1.1 Call the after-init hook, if available)
+    if (bootloader_after_init) {
+        bootloader_after_init();
+    }
+
 #ifdef CONFIG_BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP
     // If this boot is a wake up from the deep sleep then go to the short way,
     // try to load the application which worked before deep sleep.
@@ -79,7 +90,11 @@ static int selected_boot_partition(const bootloader_state_t *bs)
     if (bootloader_common_get_reset_reason(0) != DEEPSLEEP_RESET) {
         // Factory firmware.
 #ifdef CONFIG_BOOTLOADER_FACTORY_RESET
-        if (bootloader_common_check_long_hold_gpio(CONFIG_BOOTLOADER_NUM_PIN_FACTORY_RESET, CONFIG_BOOTLOADER_HOLD_TIME_GPIO) == 1) {
+        bool reset_level = false;
+#if CONFIG_BOOTLOADER_FACTORY_RESET_PIN_HIGH
+        reset_level = true;
+#endif
+        if (bootloader_common_check_long_hold_gpio_level(CONFIG_BOOTLOADER_NUM_PIN_FACTORY_RESET, CONFIG_BOOTLOADER_HOLD_TIME_GPIO, reset_level) == GPIO_LONG_HOLD) {
             ESP_LOGI(TAG, "Detect a condition of the factory reset");
             bool ota_data_erase = false;
 #ifdef CONFIG_BOOTLOADER_OTA_DATA_ERASE

components/bootloader/subproject/main/component.mk

@@ -10,8 +10,7 @@ LINKER_SCRIPTS := \
     $(COMPONENT_PATH)/ld/$(IDF_TARGET)/bootloader.rom.ld \
     $(IDF_PATH)/components/esp_rom/$(IDF_TARGET)/ld/$(IDF_TARGET).rom.ld \
     $(IDF_PATH)/components/esp_rom/$(IDF_TARGET)/ld/$(IDF_TARGET).rom.newlib-funcs.ld \
-    $(IDF_PATH)/components/esp_rom/$(IDF_TARGET)/ld/$(IDF_TARGET).rom.api.ld \
-    $(IDF_PATH)/components/$(IDF_TARGET)/ld/$(IDF_TARGET).peripherals.ld
+    $(IDF_PATH)/components/esp_rom/$(IDF_TARGET)/ld/$(IDF_TARGET).rom.api.ld
 
 # SPI driver patch for ROM is only needed in ESP32
 ifdef CONFIG_IDF_TARGET_ESP32

components/bootloader/subproject/main/ld/esp32/bootloader.ld

@@ -43,15 +43,19 @@ SECTIONS
     *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_disable .text.bootloader_random_disable)
+    *libesp_common.a:fpga_overrides.*(.literal.bootloader_fill_random .text.bootloader_fill_random)
     *libbootloader_support.a:bootloader_efuse_esp32.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_utility.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_console_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_panic.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_soc.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:esp_image_format.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_encrypt.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:flash_encryption_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_partitions.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:secure_boot_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot_signatures_bootloader.*(.literal .text .literal.* .text.*)
     *libmicro-ecc.a:*.*(.literal .text .literal.* .text.*)
     *libspi_flash.a:*.*(.literal .text .literal.* .text.*)

components/bootloader/subproject/main/ld/esp32c3/bootloader.ld

@@ -36,10 +36,13 @@ SECTIONS
     *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_console_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_panic.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_soc.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:esp_image_format.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_encrypt.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:flash_encryption_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_partitions.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:secure_boot_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot_signatures_bootloader.*(.literal .text .literal.* .text.*)
     *libmicro-ecc.a:*.*(.literal .text .literal.* .text.*)
     *libspi_flash.a:*.*(.literal .text .literal.* .text.*)
@@ -174,17 +177,17 @@ SECTIONS
 /**
  *  Appendix: Memory Usage of ROM bootloader
  *
- *  +--------+--------------+------+ 0x3FCC_B000
+ *  +--------+--------------+------+ 0x3FCC_AE00
  *  |        ^              |
  *  |        |              |
  *  |        | data/bss     |
  *  |        |              |
  *  |        v              |
- *  +------------------------------+ 0x3FCD_C910
+ *  +------------------------------+ 0x3FCD_C710
  *  |        ^              |
  *  |        |              |
  *  |        | stack        |
  *  |        |              |
  *  |        v              |
- *  +------------------------------+ 0x3FCD_E910
+ *  +------------------------------+ 0x3FCD_E710
  */

components/bootloader/subproject/main/ld/esp32c3/bootloader.rom.ld

@@ -0,0 +1 @@
+/* No definition for ESP32-C3 target */

components/bootloader/subproject/main/ld/esp32h2/bootloader.ld

@@ -0,0 +1,190 @@
+/** Simplified memory map for the bootloader.
+ *  Make sure the bootloader can load into main memory without overwriting itself.
+ *  We put 2nd bootloader in the high address space (before ROM stack/data/bss).
+ *  See memory usage for ROM bootloader at the end of this file.
+ */
+
+MEMORY
+{
+  iram_seg (RWX) :                  org = 0x403CE000, len = 0x2000
+  iram_loader_seg (RWX) :           org = 0x403D0000, len = 0x6000
+  dram_seg (RW) :                   org = 0x3FCD6000, len = 0x4000
+}
+
+/* Default entry point: */
+ENTRY(call_start_cpu0);
+
+SECTIONS
+{
+
+  .iram_loader.text :
+  {
+    . = ALIGN (16);
+    _loader_text_start = ABSOLUTE(.);
+    *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
+     *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
+    *liblog.a:(.literal .text .literal.* .text.*)
+    *libgcc.a:(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_clock_loader.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_common_loader.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_random.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_disable .text.bootloader_random_disable)
+    *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_enable .text.bootloader_random_enable)
+    *libbootloader_support.a:bootloader_efuse_esp32h2.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_utility.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_console_loader.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_panic.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_soc.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:esp_image_format.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:flash_encrypt.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:flash_partitions.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:secure_boot.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:secure_boot_signatures_bootloader.*(.literal .text .literal.* .text.*)
+    *libmicro-ecc.a:*.*(.literal .text .literal.* .text.*)
+    *libspi_flash.a:*.*(.literal .text .literal.* .text.*)
+    *libhal.a:wdt_hal_iram.*(.literal .text .literal.* .text.*)
+    *libesp_hw_support.a:rtc_clk.*(.literal .text .literal.* .text.*)
+    *libesp_hw_support.a:rtc_time.*(.literal .text .literal.* .text.*)
+    *libesp_hw_support.a:regi2c_ctrl.*(.literal .text .literal.* .text.*)
+    *libefuse.a:*.*(.literal .text .literal.* .text.*)
+    *(.fini.literal)
+    *(.fini)
+    *(.gnu.version)
+    _loader_text_end = ABSOLUTE(.);
+  } > iram_loader_seg
+
+  .iram.text :
+  {
+    . = ALIGN (16);
+    *(.entry.text)
+    *(.init.literal)
+    *(.init)
+  } > iram_seg
+
+
+  /* Shared RAM */
+  .dram0.bss (NOLOAD) :
+  {
+    . = ALIGN (8);
+    _dram_start = ABSOLUTE(.);
+    _bss_start = ABSOLUTE(.);
+    *(.dynsbss)
+    *(.sbss)
+    *(.sbss.*)
+    *(.gnu.linkonce.sb.*)
+    *(.scommon)
+    *(.sbss2)
+    *(.sbss2.*)
+    *(.gnu.linkonce.sb2.*)
+    *(.dynbss)
+    *(.bss)
+    *(.bss.*)
+    *(.gnu.linkonce.b.*)
+    *(COMMON)
+    . = ALIGN (8);
+    _bss_end = ABSOLUTE(.);
+  } > dram_seg
+
+  .dram0.data :
+  {
+    _data_start = ABSOLUTE(.);
+    *(.data)
+    *(.data.*)
+    *(.gnu.linkonce.d.*)
+    *(.data1)
+    *(.sdata)
+    *(.sdata.*)
+    *(.gnu.linkonce.s.*)
+    *(.sdata2)
+    *(.sdata2.*)
+    *(.gnu.linkonce.s2.*)
+    *(.jcr)
+    _data_end = ABSOLUTE(.);
+  } > dram_seg
+
+  .dram0.rodata :
+  {
+    _rodata_start = ABSOLUTE(.);
+    *(.rodata)
+    *(.rodata.*)
+    *(.gnu.linkonce.r.*)
+    *(.rodata1)
+    __XT_EXCEPTION_TABLE_ = ABSOLUTE(.);
+    *(.xt_except_table)
+    *(.gcc_except_table)
+    *(.gnu.linkonce.e.*)
+    *(.gnu.version_r)
+    *(.eh_frame)
+    . = (. + 3) & ~ 3;
+    /* C++ constructor and destructor tables, properly ordered: */
+    __init_array_start = ABSOLUTE(.);
+    KEEP (*crtbegin.*(.ctors))
+    KEEP (*(EXCLUDE_FILE (*crtend.*) .ctors))
+    KEEP (*(SORT(.ctors.*)))
+    KEEP (*(.ctors))
+    __init_array_end = ABSOLUTE(.);
+    KEEP (*crtbegin.*(.dtors))
+    KEEP (*(EXCLUDE_FILE (*crtend.*) .dtors))
+    KEEP (*(SORT(.dtors.*)))
+    KEEP (*(.dtors))
+    /*  C++ exception handlers table:  */
+    __XT_EXCEPTION_DESCS_ = ABSOLUTE(.);
+    *(.xt_except_desc)
+    *(.gnu.linkonce.h.*)
+    __XT_EXCEPTION_DESCS_END__ = ABSOLUTE(.);
+    *(.xt_except_desc_end)
+    *(.dynamic)
+    *(.gnu.version_d)
+    _rodata_end = ABSOLUTE(.);
+    /* Literals are also RO data. */
+    _lit4_start = ABSOLUTE(.);
+    *(*.lit4)
+    *(.lit4.*)
+    *(.gnu.linkonce.lit4.*)
+    _lit4_end = ABSOLUTE(.);
+    . = ALIGN(4);
+    _dram_end = ABSOLUTE(.);
+  } > dram_seg
+
+  .iram.text :
+  {
+    _stext = .;
+    _text_start = ABSOLUTE(.);
+    *(.literal .text .literal.* .text.* .stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
+    *(.iram .iram.*) /* catch stray IRAM_ATTR */
+    *(.fini.literal)
+    *(.fini)
+    *(.gnu.version)
+
+    /** CPU will try to prefetch up to 16 bytes of
+      * of instructions. This means that any configuration (e.g. MMU, PMS) must allow
+      * safe access to up to 16 bytes after the last real instruction, add
+      * dummy bytes to ensure this
+      */
+    . += 16;
+
+    _text_end = ABSOLUTE(.);
+    _etext = .;
+  } > iram_seg
+
+}
+
+/**
+ *  Appendix: Memory Usage of ROM bootloader
+ *
+ *  +--------+--------------+------+ 0x3FCC_B900
+ *  |        ^              |
+ *  |        |              |
+ *  |        | data/bss     |
+ *  |        |              |
+ *  |        v              |
+ *  +------------------------------+ 0x3FCD_D210
+ *  |        ^              |
+ *  |        |              |
+ *  |        | stack        |
+ *  |        |              |
+ *  |        v              |
+ *  +------------------------------+ 0x3FCD_F210
+ */

components/bootloader/subproject/main/ld/esp32s2/bootloader.ld

@@ -30,15 +30,19 @@ SECTIONS
     *libbootloader_support.a:bootloader_flash.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_disable .text.bootloader_random_disable)
+    *libesp_common.a:fpga_overrides.*(.literal.bootloader_fill_random .text.bootloader_fill_random)
     *libbootloader_support.a:bootloader_efuse_esp32s2.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_utility.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_console_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_panic.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_soc.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:esp_image_format.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_encrypt.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:flash_encryption_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_partitions.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:secure_boot_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot_signatures_bootloader.*(.literal .text .literal.* .text.*)
     *libmicro-ecc.a:*.*(.literal .text .literal.* .text.*)
     *libspi_flash.a:*.*(.literal .text .literal.* .text.*)

components/bootloader/subproject/main/ld/esp32s3/bootloader.ld

@@ -31,15 +31,19 @@ SECTIONS
     *libbootloader_support.a:bootloader_random.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_disable .text.bootloader_random_disable)
     *libbootloader_support.a:bootloader_random*.*(.literal.bootloader_random_enable .text.bootloader_random_enable)
+    *libesp_common.a:fpga_overrides.*(.literal.bootloader_fill_random .text.bootloader_fill_random)
     *libbootloader_support.a:bootloader_efuse_esp32s3.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_utility.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_sha.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_console_loader.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:bootloader_panic.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:bootloader_soc.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:esp_image_format.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_encrypt.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:flash_encryption_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:flash_partitions.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot.*(.literal .text .literal.* .text.*)
+    *libbootloader_support.a:secure_boot_secure_features.*(.literal .text .literal.* .text.*)
     *libbootloader_support.a:secure_boot_signatures_bootloader.*(.literal .text .literal.* .text.*)
     *libmicro-ecc.a:*.*(.literal .text .literal.* .text.*)
     *libspi_flash.a:*.*(.literal .text .literal.* .text.*)

components/bootloader_support/CMakeLists.txt

@@ -26,7 +26,7 @@ if(BOOTLOADER_BUILD)
     "src/bootloader_console_loader.c"
     "src/bootloader_panic.c"
     "src/${IDF_TARGET}/bootloader_sha.c"
-    "src/${IDF_TARGET}/flash_encrypt.c"
+    "src/${IDF_TARGET}/bootloader_soc.c"
     "src/${IDF_TARGET}/bootloader_${IDF_TARGET}.c"
     )
     list(APPEND priv_requires hal)
@@ -35,18 +35,32 @@ else()
         "src/idf/bootloader_sha.c")
     set(include_dirs "include")
     set(priv_include_dirs "include_bootloader")
-    set(priv_requires spi_flash mbedtls efuse app_update)
+    # heap is required for `heap_memory_layout.h` header
+    set(priv_requires spi_flash mbedtls efuse app_update heap)
 endif()
 
 if(BOOTLOADER_BUILD)
+    if(CONFIG_SECURE_FLASH_ENC_ENABLED)
+        list(APPEND srcs "src/flash_encryption/flash_encrypt.c"
+                         "src/${IDF_TARGET}/flash_encryption_secure_features.c")
+    endif()
+
     if(CONFIG_SECURE_SIGNED_ON_BOOT)
         if(CONFIG_SECURE_SIGNED_APPS_ECDSA_SCHEME)
             list(APPEND srcs "src/secure_boot_v1/secure_boot_signatures_bootloader.c")
         endif()
+        if(CONFIG_SECURE_BOOT_V1_ENABLED)
+            list(APPEND srcs "src/secure_boot_v1/secure_boot.c"
+                             "src/${IDF_TARGET}/secure_boot_secure_features.c")
+        endif()
 
         if(CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME)
             list(APPEND srcs "src/secure_boot_v2/secure_boot_signatures_bootloader.c")
         endif()
+        if(CONFIG_SECURE_BOOT_V2_ENABLED)
+            list(APPEND srcs "src/secure_boot_v2/secure_boot.c"
+                             "src/${IDF_TARGET}/secure_boot_secure_features.c")
+        endif()
     endif()
 else()
     if(CONFIG_SECURE_SIGNED_ON_UPDATE)
@@ -60,11 +74,6 @@ else()
     endif()
 endif()
 
-if(CONFIG_SECURE_BOOT AND BOOTLOADER_BUILD)
-    list(APPEND srcs
-        "src/${IDF_TARGET}/secure_boot.c")
-endif()
-
 set(requires soc) #unfortunately the header directly uses SOC registers
 
 idf_component_register(SRCS "${srcs}"

components/bootloader_support/component.mk

@@ -15,6 +15,9 @@ ifndef IS_BOOTLOADER_BUILD
 COMPONENT_SRCDIRS += src/idf  # idf sub-directory contains platform agnostic IDF versions
 else
 COMPONENT_SRCDIRS += src/$(IDF_TARGET)  # one sub-dir per chip
+ifdef CONFIG_SECURE_FLASH_ENC_ENABLED
+COMPONENT_SRCDIRS += src/flash_encryption
+endif
 endif
 
 ifndef IS_BOOTLOADER_BUILD
@@ -28,12 +31,15 @@ endif
 COMPONENT_OBJEXCLUDE += src/bootloader_flash_config_esp32s2.o \
 			src/bootloader_flash_config_esp32s3.o \
 			src/bootloader_flash_config_esp32c3.o \
+			src/bootloader_flash_config_esp32h2.o \
 			src/bootloader_efuse_esp32s2.o \
 			src/bootloader_efuse_esp32s3.o \
 			src/bootloader_efuse_esp32c3.o \
+			src/bootloader_efuse_esp32h2.o \
 			src/bootloader_random_esp32s2.o \
 			src/bootloader_random_esp32s3.o \
-			src/bootloader_random_esp32c3.o
+			src/bootloader_random_esp32c3.o \
+			src/bootloader_random_esp32h2.o
 
 ifdef IS_BOOTLOADER_BUILD
 	ifndef CONFIG_SECURE_SIGNED_APPS_ECDSA_SCHEME
@@ -43,6 +49,23 @@ ifdef IS_BOOTLOADER_BUILD
 	ifndef CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME
 		COMPONENT_OBJEXCLUDE += src/secure_boot_v2/secure_boot_signatures_bootloader.o
 	endif
+
+	ifndef CONFIG_SECURE_BOOT_V1_ENABLED
+		COMPONENT_OBJEXCLUDE += src/secure_boot_v1/secure_boot.o
+	endif
+
+	ifndef CONFIG_SECURE_BOOT_V2_ENABLED
+		COMPONENT_OBJEXCLUDE += src/secure_boot_v2/secure_boot.o
+	endif
+
+	ifndef CONFIG_SECURE_BOOT
+		COMPONENT_OBJEXCLUDE += src/${IDF_TARGET}/secure_boot_secure_features.o
+	endif
+
+	ifndef CONFIG_SECURE_FLASH_ENC_ENABLED
+		COMPONENT_OBJEXCLUDE += src/${IDF_TARGET}/flash_encryption_secure_features.o
+	endif
+
 	COMPONENT_OBJEXCLUDE += src/secure_boot_v1/secure_boot_signatures_app.o \
 				src/secure_boot_v2/secure_boot_signatures_app.o
 else
@@ -53,14 +76,13 @@ else
 	ifndef CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME
 		COMPONENT_OBJEXCLUDE += src/secure_boot_v2/secure_boot_signatures_app.o
 	endif
+
 	COMPONENT_OBJEXCLUDE += src/secure_boot_v1/secure_boot_signatures_bootloader.o \
-				src/secure_boot_v2/secure_boot_signatures_bootloader.o
+				src/secure_boot_v1/secure_boot.o \
+				src/secure_boot_v2/secure_boot_signatures_bootloader.o \
+				src/secure_boot_v2/secure_boot.o
 endif # IS_BOOTLOADER_BUILD
 
-ifndef CONFIG_SECURE_BOOT
-COMPONENT_OBJEXCLUDE += src/$(IDF_TARGET)/secure_boot.o
-endif
-
 #
 # Secure boot signing key support
 #

components/bootloader_support/include/bootloader_clock.h

@@ -1,16 +1,8 @@
-// Copyright 2017 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #pragma once
 

components/bootloader_support/include/bootloader_common.h

@@ -1,22 +1,15 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #pragma once
 #include "esp_flash_partitions.h"
 #include "esp_image_format.h"
 #include "esp_app_format.h"
-// RESET_REASON is declared in rom/rtc.h
+// [refactor-todo]: we shouldn't expose ROM header files in a public API header, remove them in v5.0
+// Tracked in IDF-1968
 #if CONFIG_IDF_TARGET_ESP32
 #include "esp32/rom/rtc.h"
 #elif CONFIG_IDF_TARGET_ESP32S2
@@ -25,6 +18,8 @@
 #include "esp32s3/rom/rtc.h"
 #elif CONFIG_IDF_TARGET_ESP32C3
 #include "esp32c3/rom/rtc.h"
+#elif CONFIG_IDF_TARGET_ESP32H2
+#include "esp32h2/rom/rtc.h"
 #endif
 
 #ifdef __cplusplus
@@ -68,18 +63,37 @@ bool bootloader_common_ota_select_valid(const esp_ota_select_entry_t *s);
 bool bootloader_common_ota_select_invalid(const esp_ota_select_entry_t *s);
 
 /**
- * @brief Check if the GPIO input is a long hold or a short hold.
+ * @brief Check if a GPIO input is held low for a long period, short period, or not
+ * at all.
+ *
+ * This function will configure the specified GPIO as an input with internal pull-up enabled.
  *
- * Number of the GPIO input will be configured as an input with internal pull-up enabled.
  * If the GPIO input is held low continuously for delay_sec period then it is a long hold.
  * If the GPIO input is held low for less period then it is a short hold.
  *
  * @param[in] num_pin Number of the GPIO input.
  * @param[in] delay_sec Input must be driven low for at least this long, continuously.
- * @return esp_comm_gpio_hold_t Defines type of hold a GPIO in low state.
+ * @return esp_comm_gpio_hold_t Type of low level hold detected, if any.
  */
 esp_comm_gpio_hold_t bootloader_common_check_long_hold_gpio(uint32_t num_pin, uint32_t delay_sec);
 
+/**
+ * @brief Check if a GPIO input is held low or high for a long period, short period, or not
+ * at all.
+ *
+ * This function will configure the specified GPIO as an input with internal pull-up enabled.
+ *
+ * If the GPIO input is held at 'level' continuously for delay_sec period then it is a long hold.
+ * If the GPIO input is held at 'level' for less period then it is a short hold.
+ *
+ * @param[in] num_pin Number of the GPIO input.
+ * @param[in] delay_sec Input must be driven to 'level' for at least this long, continuously.
+ * @param[in] level Input pin level to trigger on hold
+ * @return esp_comm_gpio_hold_t Type of hold detected, if any.
+ */
+esp_comm_gpio_hold_t bootloader_common_check_long_hold_gpio_level(uint32_t num_pin, uint32_t delay_sec, bool level);
+
+
 /**
  * @brief Erase the partition data that is specified in the transferred list.
  *
@@ -105,6 +119,15 @@ bool bootloader_common_label_search(const char *list, char *label);
  */
 void bootloader_configure_spi_pins(int drv);
 
+/**
+ * @brief Get flash CS IO
+ *
+ * Can be determined by eFuse values, or the default value
+ *
+ * @return Flash CS IO
+ */
+uint8_t bootloader_flash_get_cs_io(void);
+
 /**
  * @brief Calculates a sha-256 for a given partition or returns a appended digest.
  *

components/bootloader_support/include/bootloader_flash.h

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #pragma once
 
 #include <esp_err.h>
@@ -18,6 +10,18 @@
 #include "sdkconfig.h"
 #include "soc/soc_caps.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Read flash ID by sending RDID command (0x9F)
+ * @return flash raw ID
+ *     mfg_id = (ID >> 16) & 0xFF;
+       flash_id = ID & 0xffff;
+ */
+uint32_t bootloader_read_flash_id(void);
+
 #if SOC_CACHE_SUPPORT_WRAP
 /**
  * @brief Set the burst mode setting command for specified wrap mode.
@@ -27,3 +31,22 @@
  */
 esp_err_t bootloader_flash_wrap_set(spi_flash_wrap_mode_t mode);
 #endif
+
+/**
+  * @brief Unlock Flash write protect.
+  *        Please do not call this function in SDK.
+  *
+  * @note This can be overridden because it's attribute weak.
+  */
+esp_err_t bootloader_flash_unlock(void);
+
+/**
+ * @brief Startup flow recommended by XMC. Call at startup before any erase/write operation.
+ *
+ * @return ESP_OK When startup successfully, otherwise ESP_FAIL (indiciating you should reboot before erase/write).
+ */
+esp_err_t bootloader_flash_xmc_startup(void);
+
+#ifdef __cplusplus
+}
+#endif

components/bootloader_support/include/bootloader_flash_config.h

@@ -1,16 +1,8 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #pragma once
 

components/bootloader_support/include/bootloader_mem.h

@@ -1,16 +1,8 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #pragma once
 
 #ifdef __cplusplus

components/bootloader_support/include/bootloader_random.h

@@ -1,16 +1,8 @@
-// Copyright 2010-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2010-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #pragma once
 
@@ -21,32 +13,38 @@ extern "C" {
 #endif
 
 /**
- * @brief Enable early entropy source for RNG
+ * @brief Enable an entropy source for RNG if RF is disabled
  *
- * Uses the SAR ADC to feed entropy into the HWRNG. The ADC is put
- * into a test mode that reads the 1.1V internal reference source and
- * feeds the LSB of data into the HWRNG.
+ * The exact internal entropy source mechanism depends on the chip in use but
+ * all SoCs use the SAR ADC to continuously mix random bits (an internal
+ * noise reading) into the HWRNG. Consult the SoC Technical Reference
+ * Manual for more information.
  *
- * Can also be used from app code early during operation, if entropy
- * is required before WiFi stack is initialised. Call this function
- * from app code only if WiFi/BT are not yet enabled and I2S and SAR
- * ADC are not in use.
- *
- * Call bootloader_random_disable() when done.
+ * Can also be used from app code early during operation, if true
+ * random numbers are required before RF is initialised. Consult
+ * ESP-IDF Programming Guide "Random Number Generation" section for
+ * details.
  */
 void bootloader_random_enable(void);
 
 /**
- * @brief Disable early entropy source for RNG
+ * @brief Disable entropy source for RNG
  *
- * Disables SAR ADC source and resets the I2S hardware.
+ * Disables internal entropy source. Must be called after
+ * bootloader_random_enable() and before RF features, ADC, or
+ * I2S (ESP32 only) are initialized.
  *
+ * Consult the ESP-IDF Programming Guide "Random Number Generation"
+ * section for details.
  */
 void bootloader_random_disable(void);
 
 /**
  * @brief Fill buffer with 'length' random bytes
  *
+ * @note If this function is being called from app code only, and never
+ * from the bootloader, then it's better to call esp_fill_random().
+ *
  * @param buffer Pointer to buffer
  * @param length This many bytes of random data will be copied to buffer
  */

components/bootloader_support/include/bootloader_util.h

@@ -1,16 +1,8 @@
-// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #pragma once
 

components/bootloader_support/include/esp_app_format.h

@@ -1,16 +1,8 @@
-// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #pragma once
 
 #include <inttypes.h>
@@ -23,7 +15,8 @@ typedef enum {
     ESP_CHIP_ID_ESP32 = 0x0000,  /*!< chip ID: ESP32 */
     ESP_CHIP_ID_ESP32S2 = 0x0002,  /*!< chip ID: ESP32-S2 */
     ESP_CHIP_ID_ESP32C3 = 0x0005, /*!< chip ID: ESP32-C3 */
-    ESP_CHIP_ID_ESP32S3 = 0x0006, /*!< chip ID: ESP32-S3 */
+    ESP_CHIP_ID_ESP32S3 = 0x0009, /*!< chip ID: ESP32-S3 */
+    ESP_CHIP_ID_ESP32H2 = 0x000A, /*!< chip ID: ESP32-H2 */  // ESP32H2-TODO: IDF-3475
     ESP_CHIP_ID_INVALID = 0xFFFF /*!< Invalid chip ID (we defined it to make sure the esp_chip_id_t is 2 bytes size) */
 } __attribute__((packed)) esp_chip_id_t;
 

components/bootloader_support/include/esp_flash_data_types.h

@@ -1,2 +1,7 @@
+/*
+ * SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #warning esp_flash_data_types.h has been merged into esp_flash_partitions.h, please include esp_flash_partitions.h instead
 #include "esp_flash_partitions.h"

components/bootloader_support/include/esp_flash_encrypt.h

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #pragma once
 
 #include <stdbool.h>
@@ -22,6 +14,11 @@
 #include "soc/efuse_periph.h"
 #include "sdkconfig.h"
 
+#ifdef CONFIG_EFUSE_VIRTUAL_KEEP_IN_FLASH
+#include "esp_efuse.h"
+#include "esp_efuse_table.h"
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -51,9 +48,17 @@ static inline /** @cond */ IRAM_ATTR /** @endcond */ bool esp_flash_encryption_e
 {
     uint32_t flash_crypt_cnt = 0;
 #if CONFIG_IDF_TARGET_ESP32
-    flash_crypt_cnt = REG_GET_FIELD(EFUSE_BLK0_RDATA0_REG, EFUSE_RD_FLASH_CRYPT_CNT);
+    #ifndef CONFIG_EFUSE_VIRTUAL_KEEP_IN_FLASH
+        flash_crypt_cnt = REG_GET_FIELD(EFUSE_BLK0_RDATA0_REG, EFUSE_RD_FLASH_CRYPT_CNT);
+    #else
+        esp_efuse_read_field_blob(ESP_EFUSE_FLASH_CRYPT_CNT, &flash_crypt_cnt, ESP_EFUSE_FLASH_CRYPT_CNT[0]->bit_count);
+    #endif
 #else
-    flash_crypt_cnt = REG_GET_FIELD(EFUSE_RD_REPEAT_DATA1_REG, EFUSE_SPI_BOOT_CRYPT_CNT);
+    #ifndef CONFIG_EFUSE_VIRTUAL_KEEP_IN_FLASH
+        flash_crypt_cnt = REG_GET_FIELD(EFUSE_RD_REPEAT_DATA1_REG, EFUSE_SPI_BOOT_CRYPT_CNT);
+    #else
+        esp_efuse_read_field_blob(ESP_EFUSE_SPI_BOOT_CRYPT_CNT, &flash_crypt_cnt, ESP_EFUSE_SPI_BOOT_CRYPT_CNT[0]->bit_count);
+    #endif
 #endif
     /* __builtin_parity is in flash, so we calculate parity inline */
     bool enabled = false;
@@ -159,6 +164,23 @@ esp_flash_enc_mode_t esp_get_flash_encryption_mode(void);
  */
 void esp_flash_encryption_init_checks(void);
 
+/** @brief Set all secure eFuse features related to flash encryption
+ *
+ * @return
+ *  - ESP_OK - Successfully
+ */
+esp_err_t esp_flash_encryption_enable_secure_features(void);
+
+/** @brief Switches Flash Encryption from "Development" to "Release"
+ *
+ * If already in "Release" mode, the function will do nothing.
+ * If flash encryption efuse is not enabled yet then abort.
+ * It burns:
+ *  - "disable encrypt in dl mode"
+ *  - set FLASH_CRYPT_CNT efuse to max
+ */
+void esp_flash_encryption_set_release_mode(void);
+
 #ifdef __cplusplus
 }
 #endif

components/bootloader_support/include/esp_flash_partitions.h

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #pragma once
 
 #include "esp_err.h"
@@ -42,6 +34,9 @@ extern "C" {
 
 #define PART_FLAG_ENCRYPTED (1<<0)
 
+/* The md5sum value is found this many bytes after the ESP_PARTITION_MAGIC_MD5 offset */
+#define ESP_PARTITION_MD5_OFFSET 16
+
 /* Pre-partition table fixed flash offsets */
 #define ESP_BOOTLOADER_DIGEST_OFFSET 0x0
 #define ESP_BOOTLOADER_OFFSET CONFIG_BOOTLOADER_OFFSET_IN_FLASH  /* Offset of bootloader image. Has matching value in bootloader KConfig.projbuild file. */

components/bootloader_support/include/esp_image_format.h

@@ -1,16 +1,8 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #pragma once
 
 #include <stdbool.h>