ci: upload elf map file to s3 server to reduce artifacts size

f

.gitlab/ci/build.yml

@@ -4,7 +4,6 @@
   tags:
     - build
   variables:
-    SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
     # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
     IDF_CCACHE_ENABLE: "1"
   after_script:
@@ -12,35 +11,101 @@
     - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
   dependencies: []
 
-.build_pytest_template:
+.build_cmake_template:
   extends:
     - .build_template
     - .before_script_build_jobs
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
+  dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
+  variables:
+    LDGEN_CHECK_MAPPING: 1
   artifacts:
     paths:
       - "**/build*/size.json"
       - "**/build*/build_log.txt"
       - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # ttfw tests require elf files
       - "**/build*/*.elf"
-      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      # ttfw tests require sdkconfig file
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
+      - size_info.txt
+      # unit test specific
+      - components/idf_test/unit_test/*.yml
+    when: always
+    expire_in: 4 days
+  after_script:
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
+    # upload the binary files to s3 server
+    - echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server"
+    - shopt -s globstar
+    # use || true to bypass the no-file error
+    - zip ${CI_JOB_ID}.zip **/build*/*.bin || true
+    - zip ${CI_JOB_ID}.zip **/build*/*.elf || true
+    - zip ${CI_JOB_ID}.zip **/build*/*.map || true
+    - zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true
+    - zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true
+    - zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true
+    - zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true
+    - zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true
+    - zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true
+    - mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY}
+    - mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip
+    - echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K"
+    - echo "Please download the full binary files (including *.elf and *.map files) from the following share link"
+    # would be clean up after 4 days
+    - mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --copy-sdkconfig
+      --collect-size-info size_info.txt
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --extra-preserve-dirs
+        examples/bluetooth/esp_ble_mesh/ble_mesh_console
+        examples/bluetooth/hci/controller_hci_uart_esp32
+        examples/wifi/iperf
+
+.build_pytest_template:
+  extends:
+    - .build_cmake_template
+    - .before_script_build_jobs
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # - "**/build*/*.elf"
       - "**/build*/flasher_args.json"
       - "**/build*/flash_project_args"
       - "**/build*/config/sdkconfig.json"
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
-      - $SIZE_INFO_LOCATION
+      - list_job_*.json
+      - size_info.txt
     when: always
-    expire_in: 3 days
+    expire_in: 4 days
   script:
     # CI specific options start from "--collect-size-info xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       --pytest-apps
-      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-size-info size_info.txt
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
 
@@ -138,14 +203,35 @@ build_non_test_components_apps:
     # CI specific options start from "--collect-size-info xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
       -t all
-      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-size-info size_info.txt
       --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
 
+.build_pytest_test_apps_template:
+  extends: .build_pytest_template
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # pytest test apps requires elf files for coredump tests
+      - "**/build*/*.elf"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
+      - size_info.txt
+    when: always
+    expire_in: 4 days
+
 build_pytest_test_apps_esp32:
   extends:
-    - .build_pytest_template
+    - .build_pytest_test_apps_template
     - .rules:build:custom_test-esp32
   variables:
     IDF_TARGET: esp32
@@ -153,7 +239,7 @@ build_pytest_test_apps_esp32:
 
 build_pytest_test_apps_esp32s2:
   extends:
-    - .build_pytest_template
+    - .build_pytest_test_apps_template
     - .rules:build:custom_test-esp32s2
   variables:
     IDF_TARGET: esp32s2
@@ -161,7 +247,7 @@ build_pytest_test_apps_esp32s2:
 
 build_pytest_test_apps_esp32s3:
   extends:
-    - .build_pytest_template
+    - .build_pytest_test_apps_template
     - .rules:build:custom_test-esp32s3
   variables:
     IDF_TARGET: esp32s3
@@ -169,7 +255,7 @@ build_pytest_test_apps_esp32s3:
 
 build_pytest_test_apps_esp32c3:
   extends:
-    - .build_pytest_template
+    - .build_pytest_test_apps_template
     - .rules:build:custom_test-esp32c3
   variables:
     IDF_TARGET: esp32c3
@@ -177,7 +263,7 @@ build_pytest_test_apps_esp32c3:
 
 build_pytest_test_apps_esp32c2:
   extends:
-    - .build_pytest_template
+    - .build_pytest_test_apps_template
     - .rules:build:custom_test-esp32c2
   variables:
     IDF_TARGET: esp32c2
@@ -268,19 +354,17 @@ build_ssc_esp32s3:
 
 .build_esp_idf_tests_cmake_template:
   extends:
-    - .build_template
+    - .build_cmake_template
     - .before_script_build_jobs
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - job: fast_template_app
-      artifacts: false
   artifacts:
     paths:
       - "**/build*/size.json"
       - "**/build*/build_log.txt"
       - "**/build*/*.bin"
+      # upload to s3 server to save the artifacts size
+      # - "**/build*/*.map"
+      # ttfw tests require elf files
       - "**/build*/*.elf"
-      - "**/build*/*.map"
       - "**/build*/flasher_args.json"
       - "**/build*/flash_project_args"
       - "**/build*/config/sdkconfig.json"
@@ -288,12 +372,10 @@ build_ssc_esp32s3:
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
       - list_job_*.json
-      - $SIZE_INFO_LOCATION
+      - size_info.txt
       - components/idf_test/unit_test/*.yml
     when: always
     expire_in: 4 days
-  variables:
-    LDGEN_CHECK_MAPPING: 1
   script:
     # CI specific options start from "--collect-size-info xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
@@ -301,7 +383,7 @@ build_ssc_esp32s3:
       --config "configs/*="
       --copy-sdkconfig
       --preserve-all
-      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-size-info size_info.txt
       --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
@@ -344,47 +426,6 @@ build_esp_idf_tests_cmake_esp32c3:
   variables:
     IDF_TARGET: esp32c3
 
-.build_cmake_template:
-  extends:
-    - .build_template
-    - .before_script_build_jobs
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - job: fast_template_app
-      artifacts: false
-  variables:
-    LDGEN_CHECK_MAPPING: 1
-  artifacts:
-    paths:
-      - "**/build*/size.json"
-      - "**/build*/build_log.txt"
-      - "**/build*/*.bin"
-      - "**/build*/*.elf"
-      - "**/build*/*.map"
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/sdkconfig"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-      - list_job_*.json
-      - $SIZE_INFO_LOCATION
-    when: always
-    expire_in: 4 days
-  script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
-    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-      -t $IDF_TARGET
-      --copy-sdkconfig
-      --collect-size-info $SIZE_INFO_LOCATION
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
-      --parallel-count ${CI_NODE_TOTAL:-1}
-      --parallel-index ${CI_NODE_INDEX:-1}
-      --extra-preserve-dirs
-        examples/bluetooth/esp_ble_mesh/ble_mesh_console
-        examples/bluetooth/hci/controller_hci_uart_esp32
-        examples/wifi/iperf
-
 build_examples_cmake_esp32:
   extends:
     - .build_cmake_template