filesystem.py 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190
  1. import errno
  2. import fnmatch
  3. import os
  4. import os.path
  5. import random
  6. import sys
  7. from contextlib import contextmanager
  8. from tempfile import NamedTemporaryFile
  9. # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is
  10. # why we ignore the type on this import.
  11. from pip._vendor.retrying import retry # type: ignore
  12. from pip._vendor.six import PY2
  13. from pip._internal.utils.compat import get_path_uid
  14. from pip._internal.utils.misc import format_size
  15. from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast
  16. if MYPY_CHECK_RUNNING:
  17. from typing import Any, BinaryIO, Iterator, List, Union
  18. class NamedTemporaryFileResult(BinaryIO):
  19. @property
  20. def file(self):
  21. # type: () -> BinaryIO
  22. pass
  23. def check_path_owner(path):
  24. # type: (str) -> bool
  25. # If we don't have a way to check the effective uid of this process, then
  26. # we'll just assume that we own the directory.
  27. if sys.platform == "win32" or not hasattr(os, "geteuid"):
  28. return True
  29. assert os.path.isabs(path)
  30. previous = None
  31. while path != previous:
  32. if os.path.lexists(path):
  33. # Check if path is writable by current user.
  34. if os.geteuid() == 0:
  35. # Special handling for root user in order to handle properly
  36. # cases where users use sudo without -H flag.
  37. try:
  38. path_uid = get_path_uid(path)
  39. except OSError:
  40. return False
  41. return path_uid == 0
  42. else:
  43. return os.access(path, os.W_OK)
  44. else:
  45. previous, path = path, os.path.dirname(path)
  46. return False # assume we don't own the path
  47. @contextmanager
  48. def adjacent_tmp_file(path, **kwargs):
  49. # type: (str, **Any) -> Iterator[NamedTemporaryFileResult]
  50. """Return a file-like object pointing to a tmp file next to path.
  51. The file is created securely and is ensured to be written to disk
  52. after the context reaches its end.
  53. kwargs will be passed to tempfile.NamedTemporaryFile to control
  54. the way the temporary file will be opened.
  55. """
  56. with NamedTemporaryFile(
  57. delete=False,
  58. dir=os.path.dirname(path),
  59. prefix=os.path.basename(path),
  60. suffix='.tmp',
  61. **kwargs
  62. ) as f:
  63. result = cast('NamedTemporaryFileResult', f)
  64. try:
  65. yield result
  66. finally:
  67. result.file.flush()
  68. os.fsync(result.file.fileno())
  69. _replace_retry = retry(stop_max_delay=1000, wait_fixed=250)
  70. if PY2:
  71. @_replace_retry
  72. def replace(src, dest):
  73. # type: (str, str) -> None
  74. try:
  75. os.rename(src, dest)
  76. except OSError:
  77. os.remove(dest)
  78. os.rename(src, dest)
  79. else:
  80. replace = _replace_retry(os.replace)
  81. # test_writable_dir and _test_writable_dir_win are copied from Flit,
  82. # with the author's agreement to also place them under pip's license.
  83. def test_writable_dir(path):
  84. # type: (str) -> bool
  85. """Check if a directory is writable.
  86. Uses os.access() on POSIX, tries creating files on Windows.
  87. """
  88. # If the directory doesn't exist, find the closest parent that does.
  89. while not os.path.isdir(path):
  90. parent = os.path.dirname(path)
  91. if parent == path:
  92. break # Should never get here, but infinite loops are bad
  93. path = parent
  94. if os.name == 'posix':
  95. return os.access(path, os.W_OK)
  96. return _test_writable_dir_win(path)
  97. def _test_writable_dir_win(path):
  98. # type: (str) -> bool
  99. # os.access doesn't work on Windows: http://bugs.python.org/issue2528
  100. # and we can't use tempfile: http://bugs.python.org/issue22107
  101. basename = 'accesstest_deleteme_fishfingers_custard_'
  102. alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789'
  103. for i in range(10):
  104. name = basename + ''.join(random.choice(alphabet) for _ in range(6))
  105. file = os.path.join(path, name)
  106. try:
  107. fd = os.open(file, os.O_RDWR | os.O_CREAT | os.O_EXCL)
  108. # Python 2 doesn't support FileExistsError and PermissionError.
  109. except OSError as e:
  110. # exception FileExistsError
  111. if e.errno == errno.EEXIST:
  112. continue
  113. # exception PermissionError
  114. if e.errno == errno.EPERM or e.errno == errno.EACCES:
  115. # This could be because there's a directory with the same name.
  116. # But it's highly unlikely there's a directory called that,
  117. # so we'll assume it's because the parent dir is not writable.
  118. return False
  119. raise
  120. else:
  121. os.close(fd)
  122. os.unlink(file)
  123. return True
  124. # This should never be reached
  125. raise EnvironmentError(
  126. 'Unexpected condition testing for writable directory'
  127. )
  128. def find_files(path, pattern):
  129. # type: (str, str) -> List[str]
  130. """Returns a list of absolute paths of files beneath path, recursively,
  131. with filenames which match the UNIX-style shell glob pattern."""
  132. result = [] # type: List[str]
  133. for root, dirs, files in os.walk(path):
  134. matches = fnmatch.filter(files, pattern)
  135. result.extend(os.path.join(root, f) for f in matches)
  136. return result
  137. def file_size(path):
  138. # type: (str) -> Union[int, float]
  139. # If it's a symlink, return 0.
  140. if os.path.islink(path):
  141. return 0
  142. return os.path.getsize(path)
  143. def format_file_size(path):
  144. # type: (str) -> str
  145. return format_size(file_size(path))
  146. def directory_size(path):
  147. # type: (str) -> Union[int, float]
  148. size = 0.0
  149. for root, _dirs, files in os.walk(path):
  150. for filename in files:
  151. file_path = os.path.join(root, filename)
  152. size += file_size(file_path)
  153. return size
  154. def format_directory_size(path):
  155. # type: (str) -> str
  156. return format_size(directory_size(path))