docs: Run build_docs script in ci

.gitlab-ci.yml

@@ -6,7 +6,7 @@ stages:
   - target_test
   - post_check
   - deploy
-  - post_check
+  - post_deploy
 
 variables:
 # System environment
@@ -76,6 +76,9 @@ variables:
   rm -rf "$CUSTOM_TOOLCHAIN_PATH"
 
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
+  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
+  export IDF_MIRROR_PREFIX_MAP=
+  fi
   if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
   tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
   fi
@@ -142,7 +145,7 @@ after_script:
   tags:
     - host_test
   dependencies: []
-  extends: .before_script_lesser_nofilter
+  extends: .before_script_lesser
 
 .macos_build_template:
   stage: build
@@ -163,3 +166,4 @@ include:
   - '/tools/ci/config/target-test.yml'
   - '/tools/ci/config/post_check.yml'
   - '/tools/ci/config/deploy.yml'
+  - '/tools/ci/config/post_deploy.yml'

tools/ci/config/build.yml

@@ -186,25 +186,16 @@ build_examples_cmake_esp32s2:
 
 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
 
-build_docs:
+.build_docs_template: &build_docs_template
   stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:esp-docs-20200204-ea18dcbd
   tags:
     - build_docs
   artifacts:
     when: always
     paths:
-      # English version of documentation
-      - docs/en/doxygen-warning-log.txt
-      - docs/en/sphinx-warning-log.txt
-      - docs/en/sphinx-warning-log-sanitized.txt
-      - docs/en/_build/html
-      - docs/sphinx-err-*
-      # Chinese version of documentation
-      - docs/zh_CN/doxygen-warning-log.txt
-      - docs/zh_CN/sphinx-warning-log.txt
-      - docs/zh_CN/sphinx-warning-log-sanitized.txt
-      - docs/zh_CN/_build/html
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
     expire_in: 4 days
   only:
     variables:
@@ -212,17 +203,34 @@ build_docs:
       - $BOT_LABEL_BUILD
       - $BOT_LABEL_BUILD_DOCS
       - $BOT_LABEL_REGULAR_TEST
+  dependencies: []
   script:
     - cd docs
-    - ./check_lang_folder_sync.sh
-    - cd en
-    - make gh-linkcheck
-    - make html
-    - ../check_doc_warnings.sh
-    - cd ../zh_CN
-    - make gh-linkcheck
-    - make html
-    - ../check_doc_warnings.sh
+    - ./build_docs.py -l $DOCLANG -t $DOCTGT build
+
+build_docs_en_esp32:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32"
+
+build_docs_en_esp32s2:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32s2"
+
+build_docs_zh_CN_esp32:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32"
+
+build_docs_zh_CN_esp32s2:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32s2"
 
 test_build_system:
   extends: .build_template

tools/ci/config/deploy.yml

@@ -70,6 +70,14 @@ push_to_github:
     - git remote add github git@github.com:espressif/esp-idf.git
     - tools/ci/push_to_github.sh
 
+.upload_doc_archive: &upload_doc_archive |
+  cd docs/_build/$DOCLANG/$DOCTGT
+  mv html $GIT_VER
+  tar czf $GIT_VER.tar.gz $GIT_VER
+  scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/$DOCLANG/$DOCTGT
+  ssh $DOCS_SERVER -x "cd $DOCS_PATH/$DOCLANG/$DOCTGT && tar xzf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+  cd -
+
 deploy_docs:
   stage: deploy
   image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
@@ -86,7 +94,10 @@ deploy_docs:
       - $BOT_TRIGGER_WITH_LABEL == null
       - $BOT_LABEL_BUILD_DOCS
   dependencies:
-    - build_docs
+    - build_docs_en_esp32
+    - build_docs_en_esp32s2
+    - build_docs_zh_CN_esp32
+    - build_docs_zh_CN_esp32s2
   extends: .before_script_lesser
   script:
     - mkdir -p ~/.ssh
@@ -96,19 +107,24 @@ deploy_docs:
     - chmod 600 ~/.ssh/id_rsa
     - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
     - export GIT_VER=$(git describe --always)
-    - cd docs/en/_build/
-    - mv html $GIT_VER
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
-    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
-    - cd ../../zh_CN/_build/
-    - mv html $GIT_VER
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
-    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+
+    - DOCLANG=en; DOCTGT=esp32
+    - *upload_doc_archive
+
+    - DOCLANG=en; DOCTGT=esp32s2
+    - *upload_doc_archive
+
+    - DOCLANG=zh_CN; DOCTGT=esp32
+    - *upload_doc_archive
+
+    - DOCLANG=zh_CN; DOCTGT=esp32s2
+    - *upload_doc_archive
+
     # add link to preview doc
-    - echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
-    - echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"
+    - echo "[document preview][en][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32/${GIT_VER}/index.html"
+    - echo "[document preview][en][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32s2/${GIT_VER}/index.html"
+    - echo "[document preview][zh_CN][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32/${GIT_VER}/index.html"
+    - echo "[document preview][zh_CN][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32s2/${GIT_VER}/index.html"
 
 deploy_test_result:
   stage: deploy

tools/ci/config/host-test.yml

@@ -290,28 +290,3 @@ test_sysviewtrace_proc:
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
-
-check_doc_links:
-  stage: host_test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - check_doc_links
-  only:
-    refs:
-      # can only be triggered
-      - triggers
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD_DOCS
-  artifacts:
-    paths:
-      - docs/_build/linkcheck
-    expire_in: 1 week
-  dependencies: []
-  script:
-    # must be triggered with CHECK_LINKS=Yes, otherwise exit without test
-    - test "$CHECK_LINKS" = "Yes" || exit 0
-    # can only run on master branch (otherwise the commit is not on Github yet)
-    - test "${CI_COMMIT_REF_NAME}" = "master" || exit 0
-    - cd docs
-    - make linkcheck

tools/ci/config/post_deploy.yml

@@ -0,0 +1,42 @@
+.check_doc_links_template: &check_doc_links_template
+  stage: post_deploy
+  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:esp-docs-20200204-ea18dcbd
+  tags: [ "build", "amd64", "internet" ]
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+    expire_in: 1 week
+  allow_failure: true
+  dependencies: []
+  script:
+    - cd docs
+    - ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck
+
+check_doc_links_en_esp32:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32"
+
+check_doc_links_en_esp32s2:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32s2"
+
+check_doc_links_zh_CN_esp32:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32"
+
+check_doc_links_zh_CN_esp32s2:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32s2"

tools/ci/config/pre_check.yml

@@ -8,6 +8,24 @@ check_permissions:
   script:
     - tools/ci/check-executable.sh
 
+check_docs_lang_sync:
+  extends: .check_job_template
+  stage: pre_check
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ./check_lang_folder_sync.sh
+
+check_docs_gh_links:
+  extends: .build_docs_template
+  stage: pre_check
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ./build_docs.py gh-linkcheck
+
 check_version:
   extends: .check_job_template
   # Don't run this for feature/bugfix branches, so that it is possible to modify