base_subprocess.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284
  1. import collections
  2. import subprocess
  3. import warnings
  4. from . import protocols
  5. from . import transports
  6. from .log import logger
  7. class BaseSubprocessTransport(transports.SubprocessTransport):
  8. def __init__(self, loop, protocol, args, shell,
  9. stdin, stdout, stderr, bufsize,
  10. waiter=None, extra=None, **kwargs):
  11. super().__init__(extra)
  12. self._closed = False
  13. self._protocol = protocol
  14. self._loop = loop
  15. self._proc = None
  16. self._pid = None
  17. self._returncode = None
  18. self._exit_waiters = []
  19. self._pending_calls = collections.deque()
  20. self._pipes = {}
  21. self._finished = False
  22. if stdin == subprocess.PIPE:
  23. self._pipes[0] = None
  24. if stdout == subprocess.PIPE:
  25. self._pipes[1] = None
  26. if stderr == subprocess.PIPE:
  27. self._pipes[2] = None
  28. # Create the child process: set the _proc attribute
  29. try:
  30. self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
  31. stderr=stderr, bufsize=bufsize, **kwargs)
  32. except:
  33. self.close()
  34. raise
  35. self._pid = self._proc.pid
  36. self._extra['subprocess'] = self._proc
  37. if self._loop.get_debug():
  38. if isinstance(args, (bytes, str)):
  39. program = args
  40. else:
  41. program = args[0]
  42. logger.debug('process %r created: pid %s',
  43. program, self._pid)
  44. self._loop.create_task(self._connect_pipes(waiter))
  45. def __repr__(self):
  46. info = [self.__class__.__name__]
  47. if self._closed:
  48. info.append('closed')
  49. if self._pid is not None:
  50. info.append(f'pid={self._pid}')
  51. if self._returncode is not None:
  52. info.append(f'returncode={self._returncode}')
  53. elif self._pid is not None:
  54. info.append('running')
  55. else:
  56. info.append('not started')
  57. stdin = self._pipes.get(0)
  58. if stdin is not None:
  59. info.append(f'stdin={stdin.pipe}')
  60. stdout = self._pipes.get(1)
  61. stderr = self._pipes.get(2)
  62. if stdout is not None and stderr is stdout:
  63. info.append(f'stdout=stderr={stdout.pipe}')
  64. else:
  65. if stdout is not None:
  66. info.append(f'stdout={stdout.pipe}')
  67. if stderr is not None:
  68. info.append(f'stderr={stderr.pipe}')
  69. return '<{}>'.format(' '.join(info))
  70. def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
  71. raise NotImplementedError
  72. def set_protocol(self, protocol):
  73. self._protocol = protocol
  74. def get_protocol(self):
  75. return self._protocol
  76. def is_closing(self):
  77. return self._closed
  78. def close(self):
  79. if self._closed:
  80. return
  81. self._closed = True
  82. for proto in self._pipes.values():
  83. if proto is None:
  84. continue
  85. proto.pipe.close()
  86. if (self._proc is not None and
  87. # has the child process finished?
  88. self._returncode is None and
  89. # the child process has finished, but the
  90. # transport hasn't been notified yet?
  91. self._proc.poll() is None):
  92. if self._loop.get_debug():
  93. logger.warning('Close running child process: kill %r', self)
  94. try:
  95. self._proc.kill()
  96. except ProcessLookupError:
  97. pass
  98. # Don't clear the _proc reference yet: _post_init() may still run
  99. def __del__(self):
  100. if not self._closed:
  101. warnings.warn(f"unclosed transport {self!r}", ResourceWarning,
  102. source=self)
  103. self.close()
  104. def get_pid(self):
  105. return self._pid
  106. def get_returncode(self):
  107. return self._returncode
  108. def get_pipe_transport(self, fd):
  109. if fd in self._pipes:
  110. return self._pipes[fd].pipe
  111. else:
  112. return None
  113. def _check_proc(self):
  114. if self._proc is None:
  115. raise ProcessLookupError()
  116. def send_signal(self, signal):
  117. self._check_proc()
  118. self._proc.send_signal(signal)
  119. def terminate(self):
  120. self._check_proc()
  121. self._proc.terminate()
  122. def kill(self):
  123. self._check_proc()
  124. self._proc.kill()
  125. async def _connect_pipes(self, waiter):
  126. try:
  127. proc = self._proc
  128. loop = self._loop
  129. if proc.stdin is not None:
  130. _, pipe = await loop.connect_write_pipe(
  131. lambda: WriteSubprocessPipeProto(self, 0),
  132. proc.stdin)
  133. self._pipes[0] = pipe
  134. if proc.stdout is not None:
  135. _, pipe = await loop.connect_read_pipe(
  136. lambda: ReadSubprocessPipeProto(self, 1),
  137. proc.stdout)
  138. self._pipes[1] = pipe
  139. if proc.stderr is not None:
  140. _, pipe = await loop.connect_read_pipe(
  141. lambda: ReadSubprocessPipeProto(self, 2),
  142. proc.stderr)
  143. self._pipes[2] = pipe
  144. assert self._pending_calls is not None
  145. loop.call_soon(self._protocol.connection_made, self)
  146. for callback, data in self._pending_calls:
  147. loop.call_soon(callback, *data)
  148. self._pending_calls = None
  149. except Exception as exc:
  150. if waiter is not None and not waiter.cancelled():
  151. waiter.set_exception(exc)
  152. else:
  153. if waiter is not None and not waiter.cancelled():
  154. waiter.set_result(None)
  155. def _call(self, cb, *data):
  156. if self._pending_calls is not None:
  157. self._pending_calls.append((cb, data))
  158. else:
  159. self._loop.call_soon(cb, *data)
  160. def _pipe_connection_lost(self, fd, exc):
  161. self._call(self._protocol.pipe_connection_lost, fd, exc)
  162. self._try_finish()
  163. def _pipe_data_received(self, fd, data):
  164. self._call(self._protocol.pipe_data_received, fd, data)
  165. def _process_exited(self, returncode):
  166. assert returncode is not None, returncode
  167. assert self._returncode is None, self._returncode
  168. if self._loop.get_debug():
  169. logger.info('%r exited with return code %r', self, returncode)
  170. self._returncode = returncode
  171. if self._proc.returncode is None:
  172. # asyncio uses a child watcher: copy the status into the Popen
  173. # object. On Python 3.6, it is required to avoid a ResourceWarning.
  174. self._proc.returncode = returncode
  175. self._call(self._protocol.process_exited)
  176. self._try_finish()
  177. # wake up futures waiting for wait()
  178. for waiter in self._exit_waiters:
  179. if not waiter.cancelled():
  180. waiter.set_result(returncode)
  181. self._exit_waiters = None
  182. async def _wait(self):
  183. """Wait until the process exit and return the process return code.
  184. This method is a coroutine."""
  185. if self._returncode is not None:
  186. return self._returncode
  187. waiter = self._loop.create_future()
  188. self._exit_waiters.append(waiter)
  189. return await waiter
  190. def _try_finish(self):
  191. assert not self._finished
  192. if self._returncode is None:
  193. return
  194. if all(p is not None and p.disconnected
  195. for p in self._pipes.values()):
  196. self._finished = True
  197. self._call(self._call_connection_lost, None)
  198. def _call_connection_lost(self, exc):
  199. try:
  200. self._protocol.connection_lost(exc)
  201. finally:
  202. self._loop = None
  203. self._proc = None
  204. self._protocol = None
  205. class WriteSubprocessPipeProto(protocols.BaseProtocol):
  206. def __init__(self, proc, fd):
  207. self.proc = proc
  208. self.fd = fd
  209. self.pipe = None
  210. self.disconnected = False
  211. def connection_made(self, transport):
  212. self.pipe = transport
  213. def __repr__(self):
  214. return f'<{self.__class__.__name__} fd={self.fd} pipe={self.pipe!r}>'
  215. def connection_lost(self, exc):
  216. self.disconnected = True
  217. self.proc._pipe_connection_lost(self.fd, exc)
  218. self.proc = None
  219. def pause_writing(self):
  220. self.proc._protocol.pause_writing()
  221. def resume_writing(self):
  222. self.proc._protocol.resume_writing()
  223. class ReadSubprocessPipeProto(WriteSubprocessPipeProto,
  224. protocols.Protocol):
  225. def data_received(self, data):
  226. self.proc._pipe_data_received(self.fd, data)