|
|
@@ -1,6 +1,7 @@
|
|
|
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
|
|
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
|
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
import argparse
|
|
|
+import logging
|
|
|
import os
|
|
|
import re
|
|
|
import tarfile
|
|
|
@@ -14,12 +15,15 @@ import gitlab
|
|
|
|
|
|
TR = Callable[..., Any]
|
|
|
|
|
|
+logging.basicConfig(level=logging.INFO)
|
|
|
+
|
|
|
|
|
|
def retry(func: TR) -> TR:
|
|
|
"""
|
|
|
This wrapper will only catch several exception types associated with
|
|
|
"network issues" and retry the whole function.
|
|
|
"""
|
|
|
+
|
|
|
@wraps(func)
|
|
|
def wrapper(self: 'Gitlab', *args: Any, **kwargs: Any) -> Any:
|
|
|
retried = 0
|
|
|
@@ -41,12 +45,15 @@ def retry(func: TR) -> TR:
|
|
|
if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
|
|
|
raise e # get out of the loop
|
|
|
else:
|
|
|
- print('Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), retried))
|
|
|
+ logging.warning(
|
|
|
+ 'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
|
|
|
+ retried))
|
|
|
time.sleep(2 ** retried) # wait a bit more after each retry
|
|
|
continue
|
|
|
else:
|
|
|
break
|
|
|
return res
|
|
|
+
|
|
|
return wrapper
|
|
|
|
|
|
|
|
|
@@ -138,7 +145,7 @@ class Gitlab(object):
|
|
|
try:
|
|
|
data = job.artifact(a_path) # type: bytes
|
|
|
except gitlab.GitlabGetError as e:
|
|
|
- print("Failed to download '{}' from job {}".format(a_path, job_id))
|
|
|
+ logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
|
|
|
raise e
|
|
|
raw_data_list.append(data)
|
|
|
if destination:
|
|
|
@@ -177,7 +184,8 @@ class Gitlab(object):
|
|
|
return job_id_list
|
|
|
|
|
|
@retry
|
|
|
- def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None) -> str:
|
|
|
+ def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
|
|
|
+ cache_dir: Optional[str] = None) -> str:
|
|
|
"""
|
|
|
Download archive of certain commit of a repository and extract to destination path
|
|
|
|
|
|
@@ -195,15 +203,16 @@ class Gitlab(object):
|
|
|
local_archive_file = os.path.join(cache_dir, f'{ref}.tar.gz')
|
|
|
os.makedirs(os.path.dirname(local_archive_file), exist_ok=True)
|
|
|
if os.path.isfile(local_archive_file):
|
|
|
- print('Use cached archive file. Skipping download...')
|
|
|
+ logging.info('Use cached archive file. Skipping download...')
|
|
|
else:
|
|
|
with open(local_archive_file, 'wb') as fw:
|
|
|
try:
|
|
|
project.repository_archive(sha=ref, streamed=True, action=fw.write)
|
|
|
except gitlab.GitlabGetError as e:
|
|
|
- print('Failed to archive from project {}'.format(project_id))
|
|
|
+ logging.error('Failed to archive from project {}'.format(project_id))
|
|
|
raise e
|
|
|
- print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
|
|
|
+ logging.info('Downloaded archive size: {:.03f}MB'.format(
|
|
|
+ float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
|
|
|
|
|
|
return self.decompress_archive(local_archive_file, destination)
|
|
|
|
|
|
@@ -212,10 +221,10 @@ class Gitlab(object):
|
|
|
try:
|
|
|
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
|
|
|
except gitlab.GitlabGetError as e:
|
|
|
- print('Failed to archive from project {}'.format(project_id))
|
|
|
+ logging.error('Failed to archive from project {}'.format(project_id))
|
|
|
raise e
|
|
|
|
|
|
- print('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
|
|
|
+ logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
|
|
|
|
|
|
return self.decompress_archive(temp_file.name, destination)
|
|
|
|