ci: retry Gitlab operations on error 500

tools/ci/python_packages/gitlab_api.py

@@ -1,30 +1,66 @@
+import argparse
 import os
 import re
-import argparse
-import tempfile
 import tarfile
+import tempfile
+import time
 import zipfile
 from functools import wraps
+from typing import Any, Callable, Dict, List, Optional
 
 import gitlab
 
+TR = Callable[..., Any]
+
+
+def retry(func: TR) -> TR:
+    """
+    This wrapper will only catch several exception types associated with
+    "network issues" and retry the whole function.
+    """
+    @wraps(func)
+    def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any:
+        retried = 0
+        while True:
+            try:
+                res = func(self, *args, **kwargs)
+            except (IOError, EOFError, gitlab.exceptions.GitlabError) as e:
+                if isinstance(e, gitlab.exceptions.GitlabError) and e.response_code != 500:
+                    # Only retry on error 500
+                    raise e
+                retried += 1
+                if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
+                    raise e  # get out of the loop
+                else:
+                    print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried))
+                    time.sleep(2 ** retried)  # wait a bit more after each retry
+                    continue
+            else:
+                break
+        return res
+    return wrapper
+
 
 class Gitlab(object):
     JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?")
 
     DOWNLOAD_ERROR_MAX_RETRIES = 3
 
-    def __init__(self, project_id=None):
+    def __init__(self, project_id: Optional[int] = None):
         config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG")
         if config_data_from_env:
             # prefer to load config from env variable
-            with tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
+            with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
                 temp_file.write(config_data_from_env)
-            config_files = [temp_file.name]
+            config_files = [temp_file.name]  # type: Optional[List[str]]
         else:
             # otherwise try to use config file at local filesystem
             config_files = None
-        gitlab_id = os.getenv("LOCAL_GITLAB_HTTPS_HOST")  # if None, will use the default gitlab server
+        self._init_gitlab_inst(project_id, config_files)
+
+    @retry
+    def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None:
+        gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST')  # if None, will use the default gitlab server
         self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
         self.gitlab_inst.auth()
         if project_id:
@@ -32,7 +68,8 @@ class Gitlab(object):
         else:
             self.project = None
 
-    def get_project_id(self, name, namespace=None):
+    @retry
+    def get_project_id(self, name: str, namespace: Optional[str] = None) -> int:
         """
         search project ID by name
 
@@ -56,9 +93,10 @@ class Gitlab(object):
 
         if not res:
             raise ValueError("Can't find project")
-        return res[0]
+        return int(res[0])
 
-    def download_artifacts(self, job_id, destination):
+    @retry
+    def download_artifacts(self, job_id: int, destination: str) -> None:
         """
         download full job artifacts and extract to destination.
 
@@ -73,33 +111,8 @@ class Gitlab(object):
         with zipfile.ZipFile(temp_file.name, "r") as archive_file:
             archive_file.extractall(destination)
 
-    def retry_download(func):
-        """
-        This wrapper will only catch IOError and retry the whole function.
-
-        So only use it with download functions, read() inside and atomic
-        functions
-        """
-        @wraps(func)
-        def wrapper(self, *args, **kwargs):
-            retried = 0
-            while True:
-                try:
-                    res = func(self, *args, **kwargs)
-                except (IOError, EOFError) as e:
-                    retried += 1
-                    if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
-                        raise e  # get out of the loop
-                    else:
-                        print('Retried for the {} time'.format(retried))
-                        continue
-                else:
-                    break
-            return res
-        return wrapper
-
-    @retry_download  # type: ignore
-    def download_artifact(self, job_id, artifact_path, destination=None):
+    @retry
+    def download_artifact(self, job_id: int, artifact_path: str, destination: Optional[str] = None) -> List[bytes]:
         """
         download specific path of job artifacts and extract to destination.
 
@@ -114,7 +127,7 @@ class Gitlab(object):
 
         for a_path in artifact_path:
             try:
-                data = job.artifact(a_path)
+                data = job.artifact(a_path)  # type: bytes
             except gitlab.GitlabGetError as e:
                 print("Failed to download '{}' form job {}".format(a_path, job_id))
                 raise e
@@ -131,7 +144,8 @@ class Gitlab(object):
 
         return raw_data_list
 
-    def find_job_id(self, job_name, pipeline_id=None, job_status="success"):
+    @retry
+    def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]:
         """
         Get Job ID from job name of specific pipeline
 
@@ -153,8 +167,8 @@ class Gitlab(object):
                     job_id_list.append({"id": job.id, "parallel_num": match.group(3)})
         return job_id_list
 
-    @retry_download  # type: ignore
-    def download_archive(self, ref, destination, project_id=None):
+    @retry
+    def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None) -> str:
         """
         Download archive of certain commit of a repository and extract to destination path
 
@@ -184,7 +198,7 @@ class Gitlab(object):
         return os.path.join(os.path.realpath(destination), root_name)
 
 
-if __name__ == '__main__':
+def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("action")
     parser.add_argument("project_id", type=int)
@@ -210,3 +224,7 @@ if __name__ == '__main__':
     elif args.action == "get_project_id":
         ret = gitlab_inst.get_project_id(args.project_name)
         print("project id: {}".format(ret))
+
+
+if __name__ == '__main__':
+    main()