stop_token 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652
  1. // <stop_token> -*- C++ -*-
  2. // Copyright (C) 2019-2021 Free Software Foundation, Inc.
  3. //
  4. // This file is part of the GNU ISO C++ Library. This library is free
  5. // software; you can redistribute it and/or modify it under the
  6. // terms of the GNU General Public License as published by the
  7. // Free Software Foundation; either version 3, or (at your option)
  8. // any later version.
  9. // This library is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU General Public License for more details.
  13. // Under Section 7 of GPL version 3, you are granted additional
  14. // permissions described in the GCC Runtime Library Exception, version
  15. // 3.1, as published by the Free Software Foundation.
  16. // You should have received a copy of the GNU General Public License and
  17. // a copy of the GCC Runtime Library Exception along with this program;
  18. // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
  19. // <http://www.gnu.org/licenses/>.
  20. /** @file include/stop_token
  21. * This is a Standard C++ Library header.
  22. */
  23. #ifndef _GLIBCXX_STOP_TOKEN
  24. #define _GLIBCXX_STOP_TOKEN
  25. #if __cplusplus > 201703L
  26. #include <atomic>
  27. #include <bits/std_thread.h>
  28. #include <semaphore>
  29. #define __cpp_lib_jthread 201911L
  30. namespace std _GLIBCXX_VISIBILITY(default)
  31. {
  32. _GLIBCXX_BEGIN_NAMESPACE_VERSION
  33. /// Tag type indicating a stop_source should have no shared-stop-state.
  34. struct nostopstate_t { explicit nostopstate_t() = default; };
  35. inline constexpr nostopstate_t nostopstate{};
  36. class stop_source;
  37. /// Allow testing whether a stop request has been made on a `stop_source`.
  38. class stop_token
  39. {
  40. public:
  41. stop_token() noexcept = default;
  42. stop_token(const stop_token&) noexcept = default;
  43. stop_token(stop_token&&) noexcept = default;
  44. ~stop_token() = default;
  45. stop_token&
  46. operator=(const stop_token&) noexcept = default;
  47. stop_token&
  48. operator=(stop_token&&) noexcept = default;
  49. [[nodiscard]]
  50. bool
  51. stop_possible() const noexcept
  52. {
  53. return static_cast<bool>(_M_state) && _M_state->_M_stop_possible();
  54. }
  55. [[nodiscard]]
  56. bool
  57. stop_requested() const noexcept
  58. {
  59. return static_cast<bool>(_M_state) && _M_state->_M_stop_requested();
  60. }
  61. void
  62. swap(stop_token& __rhs) noexcept
  63. { _M_state.swap(__rhs._M_state); }
  64. [[nodiscard]]
  65. friend bool
  66. operator==(const stop_token& __a, const stop_token& __b)
  67. { return __a._M_state == __b._M_state; }
  68. friend void
  69. swap(stop_token& __lhs, stop_token& __rhs) noexcept
  70. { __lhs.swap(__rhs); }
  71. private:
  72. friend class stop_source;
  73. template<typename _Callback>
  74. friend class stop_callback;
  75. static void
  76. _S_yield() noexcept
  77. {
  78. #if defined __i386__ || defined __x86_64__
  79. __builtin_ia32_pause();
  80. #endif
  81. this_thread::yield();
  82. }
  83. #ifndef __cpp_lib_semaphore
  84. // TODO: replace this with a real implementation of std::binary_semaphore
  85. struct binary_semaphore
  86. {
  87. explicit binary_semaphore(int __d) : _M_counter(__d > 0) { }
  88. void release() { _M_counter.fetch_add(1, memory_order::release); }
  89. void acquire()
  90. {
  91. int __old = 1;
  92. while (!_M_counter.compare_exchange_weak(__old, 0,
  93. memory_order::acquire,
  94. memory_order::relaxed))
  95. {
  96. __old = 1;
  97. _S_yield();
  98. }
  99. }
  100. atomic<int> _M_counter;
  101. };
  102. #endif
  103. struct _Stop_cb
  104. {
  105. using __cb_type = void(_Stop_cb*) noexcept;
  106. __cb_type* _M_callback;
  107. _Stop_cb* _M_prev = nullptr;
  108. _Stop_cb* _M_next = nullptr;
  109. bool* _M_destroyed = nullptr;
  110. binary_semaphore _M_done{0};
  111. [[__gnu__::__nonnull__]]
  112. explicit
  113. _Stop_cb(__cb_type* __cb)
  114. : _M_callback(__cb)
  115. { }
  116. void _M_run() noexcept { _M_callback(this); }
  117. };
  118. struct _Stop_state_t
  119. {
  120. using value_type = uint32_t;
  121. static constexpr value_type _S_stop_requested_bit = 1;
  122. static constexpr value_type _S_locked_bit = 2;
  123. static constexpr value_type _S_ssrc_counter_inc = 4;
  124. std::atomic<value_type> _M_owners{1};
  125. std::atomic<value_type> _M_value{_S_ssrc_counter_inc};
  126. _Stop_cb* _M_head = nullptr;
  127. std::thread::id _M_requester;
  128. _Stop_state_t() = default;
  129. bool
  130. _M_stop_possible() noexcept
  131. {
  132. // true if a stop request has already been made or there are still
  133. // stop_source objects that would allow one to be made.
  134. return _M_value.load(memory_order::acquire) & ~_S_locked_bit;
  135. }
  136. bool
  137. _M_stop_requested() noexcept
  138. {
  139. return _M_value.load(memory_order::acquire) & _S_stop_requested_bit;
  140. }
  141. void
  142. _M_add_owner() noexcept
  143. {
  144. _M_owners.fetch_add(1, memory_order::relaxed);
  145. }
  146. void
  147. _M_release_ownership() noexcept
  148. {
  149. if (_M_owners.fetch_sub(1, memory_order::acq_rel) == 1)
  150. delete this;
  151. }
  152. void
  153. _M_add_ssrc() noexcept
  154. {
  155. _M_value.fetch_add(_S_ssrc_counter_inc, memory_order::relaxed);
  156. }
  157. void
  158. _M_sub_ssrc() noexcept
  159. {
  160. _M_value.fetch_sub(_S_ssrc_counter_inc, memory_order::release);
  161. }
  162. // Obtain lock.
  163. void
  164. _M_lock() noexcept
  165. {
  166. // Can use relaxed loads to get the current value.
  167. // The successful call to _M_try_lock is an acquire operation.
  168. auto __old = _M_value.load(memory_order::relaxed);
  169. while (!_M_try_lock(__old, memory_order::relaxed))
  170. { }
  171. }
  172. // Precondition: calling thread holds the lock.
  173. void
  174. _M_unlock() noexcept
  175. {
  176. _M_value.fetch_sub(_S_locked_bit, memory_order::release);
  177. }
  178. bool
  179. _M_request_stop() noexcept
  180. {
  181. // obtain lock and set stop_requested bit
  182. auto __old = _M_value.load(memory_order::acquire);
  183. do
  184. {
  185. if (__old & _S_stop_requested_bit) // stop request already made
  186. return false;
  187. }
  188. while (!_M_try_lock_and_stop(__old));
  189. _M_requester = this_thread::get_id();
  190. while (_M_head)
  191. {
  192. bool __last_cb;
  193. _Stop_cb* __cb = _M_head;
  194. _M_head = _M_head->_M_next;
  195. if (_M_head)
  196. {
  197. _M_head->_M_prev = nullptr;
  198. __last_cb = false;
  199. }
  200. else
  201. __last_cb = true;
  202. // Allow other callbacks to be unregistered while __cb runs.
  203. _M_unlock();
  204. bool __destroyed = false;
  205. __cb->_M_destroyed = &__destroyed;
  206. // run callback
  207. __cb->_M_run();
  208. if (!__destroyed)
  209. {
  210. __cb->_M_destroyed = nullptr;
  211. // synchronize with destructor of stop_callback that owns *__cb
  212. if (!__gnu_cxx::__is_single_threaded())
  213. __cb->_M_done.release();
  214. }
  215. // Avoid relocking if we already know there are no more callbacks.
  216. if (__last_cb)
  217. return true;
  218. _M_lock();
  219. }
  220. _M_unlock();
  221. return true;
  222. }
  223. [[__gnu__::__nonnull__]]
  224. bool
  225. _M_register_callback(_Stop_cb* __cb) noexcept
  226. {
  227. auto __old = _M_value.load(memory_order::acquire);
  228. do
  229. {
  230. if (__old & _S_stop_requested_bit) // stop request already made
  231. {
  232. __cb->_M_run(); // run synchronously
  233. return false;
  234. }
  235. if (__old < _S_ssrc_counter_inc) // no stop_source owns *this
  236. // No need to register callback if no stop request can be made.
  237. // Returning false also means the stop_callback does not share
  238. // ownership of this state, but that's not observable.
  239. return false;
  240. }
  241. while (!_M_try_lock(__old));
  242. __cb->_M_next = _M_head;
  243. if (_M_head)
  244. {
  245. _M_head->_M_prev = __cb;
  246. }
  247. _M_head = __cb;
  248. _M_unlock();
  249. return true;
  250. }
  251. // Called by ~stop_callback just before destroying *__cb.
  252. [[__gnu__::__nonnull__]]
  253. void
  254. _M_remove_callback(_Stop_cb* __cb)
  255. {
  256. _M_lock();
  257. if (__cb == _M_head)
  258. {
  259. _M_head = _M_head->_M_next;
  260. if (_M_head)
  261. _M_head->_M_prev = nullptr;
  262. _M_unlock();
  263. return;
  264. }
  265. else if (__cb->_M_prev)
  266. {
  267. __cb->_M_prev->_M_next = __cb->_M_next;
  268. if (__cb->_M_next)
  269. __cb->_M_next->_M_prev = __cb->_M_prev;
  270. _M_unlock();
  271. return;
  272. }
  273. _M_unlock();
  274. // Callback is not in the list, so must have been removed by a call to
  275. // _M_request_stop.
  276. // Despite appearances there is no data race on _M_requester. The only
  277. // write to it happens before the callback is removed from the list,
  278. // and removing it from the list happens before this read.
  279. if (!(_M_requester == this_thread::get_id()))
  280. {
  281. // Synchronize with completion of callback.
  282. __cb->_M_done.acquire();
  283. // Safe for ~stop_callback to destroy *__cb now.
  284. return;
  285. }
  286. if (__cb->_M_destroyed)
  287. *__cb->_M_destroyed = true;
  288. }
  289. // Try to obtain the lock.
  290. // Returns true if the lock is acquired (with memory order acquire).
  291. // Otherwise, sets __curval = _M_value.load(__failure) and returns false.
  292. // Might fail spuriously, so must be called in a loop.
  293. bool
  294. _M_try_lock(value_type& __curval,
  295. memory_order __failure = memory_order::acquire) noexcept
  296. {
  297. return _M_do_try_lock(__curval, 0, memory_order::acquire, __failure);
  298. }
  299. // Try to obtain the lock to make a stop request.
  300. // Returns true if the lock is acquired and the _S_stop_requested_bit is
  301. // set (with memory order acq_rel so that other threads see the request).
  302. // Otherwise, sets __curval = _M_value.load(memory_order::acquire) and
  303. // returns false.
  304. // Might fail spuriously, so must be called in a loop.
  305. bool
  306. _M_try_lock_and_stop(value_type& __curval) noexcept
  307. {
  308. return _M_do_try_lock(__curval, _S_stop_requested_bit,
  309. memory_order::acq_rel, memory_order::acquire);
  310. }
  311. bool
  312. _M_do_try_lock(value_type& __curval, value_type __newbits,
  313. memory_order __success, memory_order __failure) noexcept
  314. {
  315. if (__curval & _S_locked_bit)
  316. {
  317. _S_yield();
  318. __curval = _M_value.load(__failure);
  319. return false;
  320. }
  321. __newbits |= _S_locked_bit;
  322. return _M_value.compare_exchange_weak(__curval, __curval | __newbits,
  323. __success, __failure);
  324. }
  325. };
  326. struct _Stop_state_ref
  327. {
  328. _Stop_state_ref() = default;
  329. explicit
  330. _Stop_state_ref(const stop_source&)
  331. : _M_ptr(new _Stop_state_t())
  332. { }
  333. _Stop_state_ref(const _Stop_state_ref& __other) noexcept
  334. : _M_ptr(__other._M_ptr)
  335. {
  336. if (_M_ptr)
  337. _M_ptr->_M_add_owner();
  338. }
  339. _Stop_state_ref(_Stop_state_ref&& __other) noexcept
  340. : _M_ptr(__other._M_ptr)
  341. {
  342. __other._M_ptr = nullptr;
  343. }
  344. _Stop_state_ref&
  345. operator=(const _Stop_state_ref& __other) noexcept
  346. {
  347. if (auto __ptr = __other._M_ptr; __ptr != _M_ptr)
  348. {
  349. if (__ptr)
  350. __ptr->_M_add_owner();
  351. if (_M_ptr)
  352. _M_ptr->_M_release_ownership();
  353. _M_ptr = __ptr;
  354. }
  355. return *this;
  356. }
  357. _Stop_state_ref&
  358. operator=(_Stop_state_ref&& __other) noexcept
  359. {
  360. _Stop_state_ref(std::move(__other)).swap(*this);
  361. return *this;
  362. }
  363. ~_Stop_state_ref()
  364. {
  365. if (_M_ptr)
  366. _M_ptr->_M_release_ownership();
  367. }
  368. void
  369. swap(_Stop_state_ref& __other) noexcept
  370. { std::swap(_M_ptr, __other._M_ptr); }
  371. explicit operator bool() const noexcept { return _M_ptr != nullptr; }
  372. _Stop_state_t* operator->() const noexcept { return _M_ptr; }
  373. #if __cpp_impl_three_way_comparison >= 201907L
  374. friend bool
  375. operator==(const _Stop_state_ref&, const _Stop_state_ref&) = default;
  376. #else
  377. friend bool
  378. operator==(const _Stop_state_ref& __lhs, const _Stop_state_ref& __rhs)
  379. noexcept
  380. { return __lhs._M_ptr == __rhs._M_ptr; }
  381. friend bool
  382. operator!=(const _Stop_state_ref& __lhs, const _Stop_state_ref& __rhs)
  383. noexcept
  384. { return __lhs._M_ptr != __rhs._M_ptr; }
  385. #endif
  386. private:
  387. _Stop_state_t* _M_ptr = nullptr;
  388. };
  389. _Stop_state_ref _M_state;
  390. explicit
  391. stop_token(const _Stop_state_ref& __state) noexcept
  392. : _M_state{__state}
  393. { }
  394. };
  395. /// A type that allows a stop request to be made.
  396. class stop_source
  397. {
  398. public:
  399. stop_source() : _M_state(*this)
  400. { }
  401. explicit stop_source(std::nostopstate_t) noexcept
  402. { }
  403. stop_source(const stop_source& __other) noexcept
  404. : _M_state(__other._M_state)
  405. {
  406. if (_M_state)
  407. _M_state->_M_add_ssrc();
  408. }
  409. stop_source(stop_source&&) noexcept = default;
  410. stop_source&
  411. operator=(const stop_source& __other) noexcept
  412. {
  413. if (_M_state != __other._M_state)
  414. {
  415. stop_source __sink(std::move(*this));
  416. _M_state = __other._M_state;
  417. if (_M_state)
  418. _M_state->_M_add_ssrc();
  419. }
  420. return *this;
  421. }
  422. stop_source&
  423. operator=(stop_source&&) noexcept = default;
  424. ~stop_source()
  425. {
  426. if (_M_state)
  427. _M_state->_M_sub_ssrc();
  428. }
  429. [[nodiscard]]
  430. bool
  431. stop_possible() const noexcept
  432. {
  433. return static_cast<bool>(_M_state);
  434. }
  435. [[nodiscard]]
  436. bool
  437. stop_requested() const noexcept
  438. {
  439. return static_cast<bool>(_M_state) && _M_state->_M_stop_requested();
  440. }
  441. bool
  442. request_stop() const noexcept
  443. {
  444. if (stop_possible())
  445. return _M_state->_M_request_stop();
  446. return false;
  447. }
  448. [[nodiscard]]
  449. stop_token
  450. get_token() const noexcept
  451. {
  452. return stop_token{_M_state};
  453. }
  454. void
  455. swap(stop_source& __other) noexcept
  456. {
  457. _M_state.swap(__other._M_state);
  458. }
  459. [[nodiscard]]
  460. friend bool
  461. operator==(const stop_source& __a, const stop_source& __b) noexcept
  462. {
  463. return __a._M_state == __b._M_state;
  464. }
  465. friend void
  466. swap(stop_source& __lhs, stop_source& __rhs) noexcept
  467. {
  468. __lhs.swap(__rhs);
  469. }
  470. private:
  471. stop_token::_Stop_state_ref _M_state;
  472. };
  473. /// A wrapper for callbacks to be run when a stop request is made.
  474. template<typename _Callback>
  475. class [[nodiscard]] stop_callback
  476. {
  477. static_assert(is_nothrow_destructible_v<_Callback>);
  478. static_assert(is_invocable_v<_Callback>);
  479. public:
  480. using callback_type = _Callback;
  481. template<typename _Cb,
  482. enable_if_t<is_constructible_v<_Callback, _Cb>, int> = 0>
  483. explicit
  484. stop_callback(const stop_token& __token, _Cb&& __cb)
  485. noexcept(is_nothrow_constructible_v<_Callback, _Cb>)
  486. : _M_cb(std::forward<_Cb>(__cb))
  487. {
  488. if (auto __state = __token._M_state)
  489. {
  490. if (__state->_M_register_callback(&_M_cb))
  491. _M_state.swap(__state);
  492. }
  493. }
  494. template<typename _Cb,
  495. enable_if_t<is_constructible_v<_Callback, _Cb>, int> = 0>
  496. explicit
  497. stop_callback(stop_token&& __token, _Cb&& __cb)
  498. noexcept(is_nothrow_constructible_v<_Callback, _Cb>)
  499. : _M_cb(std::forward<_Cb>(__cb))
  500. {
  501. if (auto& __state = __token._M_state)
  502. {
  503. if (__state->_M_register_callback(&_M_cb))
  504. _M_state.swap(__state);
  505. }
  506. }
  507. ~stop_callback()
  508. {
  509. if (_M_state)
  510. {
  511. _M_state->_M_remove_callback(&_M_cb);
  512. }
  513. }
  514. stop_callback(const stop_callback&) = delete;
  515. stop_callback& operator=(const stop_callback&) = delete;
  516. stop_callback(stop_callback&&) = delete;
  517. stop_callback& operator=(stop_callback&&) = delete;
  518. private:
  519. struct _Cb_impl : stop_token::_Stop_cb
  520. {
  521. template<typename _Cb>
  522. explicit
  523. _Cb_impl(_Cb&& __cb)
  524. : _Stop_cb(&_S_execute),
  525. _M_cb(std::forward<_Cb>(__cb))
  526. { }
  527. _Callback _M_cb;
  528. [[__gnu__::__nonnull__]]
  529. static void
  530. _S_execute(_Stop_cb* __that) noexcept
  531. {
  532. _Callback& __cb = static_cast<_Cb_impl*>(__that)->_M_cb;
  533. std::forward<_Callback>(__cb)();
  534. }
  535. };
  536. _Cb_impl _M_cb;
  537. stop_token::_Stop_state_ref _M_state;
  538. };
  539. template<typename _Callback>
  540. stop_callback(stop_token, _Callback) -> stop_callback<_Callback>;
  541. _GLIBCXX_END_NAMESPACE_VERSION
  542. } // namespace
  543. #endif // __cplusplus > 201703L
  544. #endif // _GLIBCXX_STOP_TOKEN