Merge branch 'ci/fix_nightly_run_missing_app' into 'master'

Ci/fix nightly run missing app

Closes IDFCI-1367 and IDFCI-1359

See merge request espressif/esp-idf!18915

conftest.py

@@ -21,7 +21,7 @@ from fnmatch import fnmatch
 from typing import Callable, List, Optional, Tuple
 
 import pytest
-from _pytest.config import Config
+from _pytest.config import Config, ExitCode
 from _pytest.fixtures import FixtureRequest
 from _pytest.main import Session
 from _pytest.nodes import Item
@@ -399,8 +399,11 @@ class IdfPytestEmbedded:
             xml.write(junit)
 
     def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None:
-        if exitstatus != 0 and self.known_failure_cases and not self.failed_cases:
-            session.exitstatus = 0
+        if exitstatus != 0:
+            if exitstatus == ExitCode.NO_TESTS_COLLECTED:
+                session.exitstatus = 0
+            elif self.known_failure_cases and not self.failed_cases:
+                session.exitstatus = 0
 
     def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None:
         if self.known_failure_cases:

tools/ci/ci_build_apps.py

@@ -26,11 +26,19 @@ def get_pytest_apps(
     pytest_cases = get_pytest_cases(paths, target, marker_expr)
 
     _paths: Set[str] = set()
-    app_configs = defaultdict(set)
+    test_related_app_configs = defaultdict(set)
     for case in pytest_cases:
         for app in case.apps:
             _paths.add(app.path)
-            app_configs[app.path].add(app.config)
+
+            if os.getenv('INCLUDE_NIGHTLY_RUN') == '1':
+                test_related_app_configs[app.path].add(app.config)
+            elif os.getenv('NIGHTLY_RUN') == '1':
+                if case.nightly_run:
+                    test_related_app_configs[app.path].add(app.config)
+            else:
+                if not case.nightly_run:
+                    test_related_app_configs[app.path].add(app.config)
 
     app_dirs = list(_paths)
     if not app_dirs:
@@ -53,7 +61,7 @@ def get_pytest_apps(
     )
 
     for app in apps:
-        is_test_related = app.config_name in app_configs[app.app_dir]
+        is_test_related = app.config_name in test_related_app_configs[app.app_dir]
         if not preserve_all and not is_test_related:
             app.preserve = False
 

tools/ci/idf_ci_utils.py

@@ -102,7 +102,7 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> List[str]:
         # this is a workaround when using under worktree
         # if you're using worktree, when running git commit a new environment variable GIT_DIR would be declared,
         # the value should be <origin_repo_path>/.git/worktrees/<worktree name>
-        # This would effect the return value of `git ls-files`, unset this would use the `cwd`value or its parent
+        # This would affect the return value of `git ls-files`, unset this would use the `cwd`value or its parent
         # folder if no `.git` folder found in `cwd`.
         workaround_env = os.environ.copy()
         workaround_env.pop('GIT_DIR', None)
@@ -150,9 +150,10 @@ class PytestCase:
     path: str
     name: str
     apps: Set[PytestApp]
+    nightly_run: bool
 
     def __hash__(self) -> int:
-        return hash((self.path, self.name, self.apps))
+        return hash((self.path, self.name, self.apps, self.nightly_run))
 
 
 class PytestCollectPlugin:
@@ -201,7 +202,14 @@ class PytestCollectPlugin:
             for i in range(count):
                 case_apps.add(PytestApp(app_paths[i], targets[i], configs[i]))
 
-            self.cases.append(PytestCase(case_path, case_name, case_apps))
+            self.cases.append(
+                PytestCase(
+                    case_path,
+                    case_name,
+                    case_apps,
+                    'nightly_run' in [marker.name for marker in item.iter_markers()],
+                )
+            )
 
 
 def get_pytest_cases(
@@ -215,6 +223,21 @@ def get_pytest_cases(
     else:
         targets = [target]
 
+    paths = to_list(paths)
+
+    origin_include_nightly_run_env = os.getenv('INCLUDE_NIGHTLY_RUN')
+    origin_nightly_run_env = os.getenv('NIGHTLY_RUN')
+
+    # disable the env vars to get all test cases
+    if 'INCLUDE_NIGHTLY_RUN' in os.environ:
+        os.environ.pop('INCLUDE_NIGHTLY_RUN')
+
+    if 'NIGHTLY_RUN' in os.environ:
+        os.environ.pop('NIGHTLY_RUN')
+
+    # collect all cases
+    os.environ['INCLUDE_NIGHTLY_RUN'] = '1'
+
     cases = []
     for t in targets:
         collector = PytestCollectPlugin(t)
@@ -241,6 +264,13 @@ def get_pytest_cases(
 
         cases.extend(collector.cases)
 
+    # revert back the env vars
+    if origin_include_nightly_run_env is not None:
+        os.environ['INCLUDE_NIGHTLY_RUN'] = origin_include_nightly_run_env
+
+    if origin_nightly_run_env is not None:
+        os.environ['NIGHTLY_RUN'] = origin_nightly_run_env
+
     return cases
 
 

tools/ci/utils.sh

@@ -75,7 +75,7 @@ function run_cmd() {
   if [[ $ret -eq 0 ]]; then
     info "==> '\$ ${cmd}' succeeded in ${runtime} seconds."
     return 0
-  echo
+  else
     error "==> '\$ ${cmd}' failed (${ret}) in ${runtime} seconds."
     return $ret
   fi