__init__.py 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299
  1. '''This module implements specialized container datatypes providing
  2. alternatives to Python's general purpose built-in containers, dict,
  3. list, set, and tuple.
  4. * namedtuple factory function for creating tuple subclasses with named fields
  5. * deque list-like container with fast appends and pops on either end
  6. * ChainMap dict-like class for creating a single view of multiple mappings
  7. * Counter dict subclass for counting hashable objects
  8. * OrderedDict dict subclass that remembers the order entries were added
  9. * defaultdict dict subclass that calls a factory function to supply missing values
  10. * UserDict wrapper around dictionary objects for easier dict subclassing
  11. * UserList wrapper around list objects for easier list subclassing
  12. * UserString wrapper around string objects for easier string subclassing
  13. '''
  14. __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList',
  15. 'UserString', 'Counter', 'OrderedDict', 'ChainMap']
  16. import _collections_abc
  17. from operator import itemgetter as _itemgetter, eq as _eq
  18. from keyword import iskeyword as _iskeyword
  19. import sys as _sys
  20. import heapq as _heapq
  21. from _weakref import proxy as _proxy
  22. from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
  23. from reprlib import recursive_repr as _recursive_repr
  24. try:
  25. from _collections import deque
  26. except ImportError:
  27. pass
  28. else:
  29. _collections_abc.MutableSequence.register(deque)
  30. try:
  31. from _collections import defaultdict
  32. except ImportError:
  33. pass
  34. def __getattr__(name):
  35. # For backwards compatibility, continue to make the collections ABCs
  36. # through Python 3.6 available through the collections module.
  37. # Note, no new collections ABCs were added in Python 3.7
  38. if name in _collections_abc.__all__:
  39. obj = getattr(_collections_abc, name)
  40. import warnings
  41. warnings.warn("Using or importing the ABCs from 'collections' instead "
  42. "of from 'collections.abc' is deprecated since Python 3.3,"
  43. "and in 3.9 it will stop working",
  44. DeprecationWarning, stacklevel=2)
  45. globals()[name] = obj
  46. return obj
  47. raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
  48. ################################################################################
  49. ### OrderedDict
  50. ################################################################################
  51. class _OrderedDictKeysView(_collections_abc.KeysView):
  52. def __reversed__(self):
  53. yield from reversed(self._mapping)
  54. class _OrderedDictItemsView(_collections_abc.ItemsView):
  55. def __reversed__(self):
  56. for key in reversed(self._mapping):
  57. yield (key, self._mapping[key])
  58. class _OrderedDictValuesView(_collections_abc.ValuesView):
  59. def __reversed__(self):
  60. for key in reversed(self._mapping):
  61. yield self._mapping[key]
  62. class _Link(object):
  63. __slots__ = 'prev', 'next', 'key', '__weakref__'
  64. class OrderedDict(dict):
  65. 'Dictionary that remembers insertion order'
  66. # An inherited dict maps keys to values.
  67. # The inherited dict provides __getitem__, __len__, __contains__, and get.
  68. # The remaining methods are order-aware.
  69. # Big-O running times for all methods are the same as regular dictionaries.
  70. # The internal self.__map dict maps keys to links in a doubly linked list.
  71. # The circular doubly linked list starts and ends with a sentinel element.
  72. # The sentinel element never gets deleted (this simplifies the algorithm).
  73. # The sentinel is in self.__hardroot with a weakref proxy in self.__root.
  74. # The prev links are weakref proxies (to prevent circular references).
  75. # Individual links are kept alive by the hard reference in self.__map.
  76. # Those hard references disappear when a key is deleted from an OrderedDict.
  77. def __init__(*args, **kwds):
  78. '''Initialize an ordered dictionary. The signature is the same as
  79. regular dictionaries. Keyword argument order is preserved.
  80. '''
  81. if not args:
  82. raise TypeError("descriptor '__init__' of 'OrderedDict' object "
  83. "needs an argument")
  84. self, *args = args
  85. if len(args) > 1:
  86. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  87. try:
  88. self.__root
  89. except AttributeError:
  90. self.__hardroot = _Link()
  91. self.__root = root = _proxy(self.__hardroot)
  92. root.prev = root.next = root
  93. self.__map = {}
  94. self.__update(*args, **kwds)
  95. def __setitem__(self, key, value,
  96. dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
  97. 'od.__setitem__(i, y) <==> od[i]=y'
  98. # Setting a new item creates a new link at the end of the linked list,
  99. # and the inherited dictionary is updated with the new key/value pair.
  100. if key not in self:
  101. self.__map[key] = link = Link()
  102. root = self.__root
  103. last = root.prev
  104. link.prev, link.next, link.key = last, root, key
  105. last.next = link
  106. root.prev = proxy(link)
  107. dict_setitem(self, key, value)
  108. def __delitem__(self, key, dict_delitem=dict.__delitem__):
  109. 'od.__delitem__(y) <==> del od[y]'
  110. # Deleting an existing item uses self.__map to find the link which gets
  111. # removed by updating the links in the predecessor and successor nodes.
  112. dict_delitem(self, key)
  113. link = self.__map.pop(key)
  114. link_prev = link.prev
  115. link_next = link.next
  116. link_prev.next = link_next
  117. link_next.prev = link_prev
  118. link.prev = None
  119. link.next = None
  120. def __iter__(self):
  121. 'od.__iter__() <==> iter(od)'
  122. # Traverse the linked list in order.
  123. root = self.__root
  124. curr = root.next
  125. while curr is not root:
  126. yield curr.key
  127. curr = curr.next
  128. def __reversed__(self):
  129. 'od.__reversed__() <==> reversed(od)'
  130. # Traverse the linked list in reverse order.
  131. root = self.__root
  132. curr = root.prev
  133. while curr is not root:
  134. yield curr.key
  135. curr = curr.prev
  136. def clear(self):
  137. 'od.clear() -> None. Remove all items from od.'
  138. root = self.__root
  139. root.prev = root.next = root
  140. self.__map.clear()
  141. dict.clear(self)
  142. def popitem(self, last=True):
  143. '''Remove and return a (key, value) pair from the dictionary.
  144. Pairs are returned in LIFO order if last is true or FIFO order if false.
  145. '''
  146. if not self:
  147. raise KeyError('dictionary is empty')
  148. root = self.__root
  149. if last:
  150. link = root.prev
  151. link_prev = link.prev
  152. link_prev.next = root
  153. root.prev = link_prev
  154. else:
  155. link = root.next
  156. link_next = link.next
  157. root.next = link_next
  158. link_next.prev = root
  159. key = link.key
  160. del self.__map[key]
  161. value = dict.pop(self, key)
  162. return key, value
  163. def move_to_end(self, key, last=True):
  164. '''Move an existing element to the end (or beginning if last is false).
  165. Raise KeyError if the element does not exist.
  166. '''
  167. link = self.__map[key]
  168. link_prev = link.prev
  169. link_next = link.next
  170. soft_link = link_next.prev
  171. link_prev.next = link_next
  172. link_next.prev = link_prev
  173. root = self.__root
  174. if last:
  175. last = root.prev
  176. link.prev = last
  177. link.next = root
  178. root.prev = soft_link
  179. last.next = link
  180. else:
  181. first = root.next
  182. link.prev = root
  183. link.next = first
  184. first.prev = soft_link
  185. root.next = link
  186. def __sizeof__(self):
  187. sizeof = _sys.getsizeof
  188. n = len(self) + 1 # number of links including root
  189. size = sizeof(self.__dict__) # instance dictionary
  190. size += sizeof(self.__map) * 2 # internal dict and inherited dict
  191. size += sizeof(self.__hardroot) * n # link objects
  192. size += sizeof(self.__root) * n # proxy objects
  193. return size
  194. update = __update = _collections_abc.MutableMapping.update
  195. def keys(self):
  196. "D.keys() -> a set-like object providing a view on D's keys"
  197. return _OrderedDictKeysView(self)
  198. def items(self):
  199. "D.items() -> a set-like object providing a view on D's items"
  200. return _OrderedDictItemsView(self)
  201. def values(self):
  202. "D.values() -> an object providing a view on D's values"
  203. return _OrderedDictValuesView(self)
  204. __ne__ = _collections_abc.MutableMapping.__ne__
  205. __marker = object()
  206. def pop(self, key, default=__marker):
  207. '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
  208. value. If key is not found, d is returned if given, otherwise KeyError
  209. is raised.
  210. '''
  211. if key in self:
  212. result = self[key]
  213. del self[key]
  214. return result
  215. if default is self.__marker:
  216. raise KeyError(key)
  217. return default
  218. def setdefault(self, key, default=None):
  219. '''Insert key with a value of default if key is not in the dictionary.
  220. Return the value for key if key is in the dictionary, else default.
  221. '''
  222. if key in self:
  223. return self[key]
  224. self[key] = default
  225. return default
  226. @_recursive_repr()
  227. def __repr__(self):
  228. 'od.__repr__() <==> repr(od)'
  229. if not self:
  230. return '%s()' % (self.__class__.__name__,)
  231. return '%s(%r)' % (self.__class__.__name__, list(self.items()))
  232. def __reduce__(self):
  233. 'Return state information for pickling'
  234. inst_dict = vars(self).copy()
  235. for k in vars(OrderedDict()):
  236. inst_dict.pop(k, None)
  237. return self.__class__, (), inst_dict or None, None, iter(self.items())
  238. def copy(self):
  239. 'od.copy() -> a shallow copy of od'
  240. return self.__class__(self)
  241. @classmethod
  242. def fromkeys(cls, iterable, value=None):
  243. '''Create a new ordered dictionary with keys from iterable and values set to value.
  244. '''
  245. self = cls()
  246. for key in iterable:
  247. self[key] = value
  248. return self
  249. def __eq__(self, other):
  250. '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
  251. while comparison to a regular mapping is order-insensitive.
  252. '''
  253. if isinstance(other, OrderedDict):
  254. return dict.__eq__(self, other) and all(map(_eq, self, other))
  255. return dict.__eq__(self, other)
  256. try:
  257. from _collections import OrderedDict
  258. except ImportError:
  259. # Leave the pure Python version in place.
  260. pass
  261. ################################################################################
  262. ### namedtuple
  263. ################################################################################
  264. _nt_itemgetters = {}
  265. def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None):
  266. """Returns a new subclass of tuple with named fields.
  267. >>> Point = namedtuple('Point', ['x', 'y'])
  268. >>> Point.__doc__ # docstring for the new class
  269. 'Point(x, y)'
  270. >>> p = Point(11, y=22) # instantiate with positional args or keywords
  271. >>> p[0] + p[1] # indexable like a plain tuple
  272. 33
  273. >>> x, y = p # unpack like a regular tuple
  274. >>> x, y
  275. (11, 22)
  276. >>> p.x + p.y # fields also accessible by name
  277. 33
  278. >>> d = p._asdict() # convert to a dictionary
  279. >>> d['x']
  280. 11
  281. >>> Point(**d) # convert from a dictionary
  282. Point(x=11, y=22)
  283. >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields
  284. Point(x=100, y=22)
  285. """
  286. # Validate the field names. At the user's option, either generate an error
  287. # message or automatically replace the field name with a valid name.
  288. if isinstance(field_names, str):
  289. field_names = field_names.replace(',', ' ').split()
  290. field_names = list(map(str, field_names))
  291. typename = _sys.intern(str(typename))
  292. if rename:
  293. seen = set()
  294. for index, name in enumerate(field_names):
  295. if (not name.isidentifier()
  296. or _iskeyword(name)
  297. or name.startswith('_')
  298. or name in seen):
  299. field_names[index] = f'_{index}'
  300. seen.add(name)
  301. for name in [typename] + field_names:
  302. if type(name) is not str:
  303. raise TypeError('Type names and field names must be strings')
  304. if not name.isidentifier():
  305. raise ValueError('Type names and field names must be valid '
  306. f'identifiers: {name!r}')
  307. if _iskeyword(name):
  308. raise ValueError('Type names and field names cannot be a '
  309. f'keyword: {name!r}')
  310. seen = set()
  311. for name in field_names:
  312. if name.startswith('_') and not rename:
  313. raise ValueError('Field names cannot start with an underscore: '
  314. f'{name!r}')
  315. if name in seen:
  316. raise ValueError(f'Encountered duplicate field name: {name!r}')
  317. seen.add(name)
  318. field_defaults = {}
  319. if defaults is not None:
  320. defaults = tuple(defaults)
  321. if len(defaults) > len(field_names):
  322. raise TypeError('Got more default values than field names')
  323. field_defaults = dict(reversed(list(zip(reversed(field_names),
  324. reversed(defaults)))))
  325. # Variables used in the methods and docstrings
  326. field_names = tuple(map(_sys.intern, field_names))
  327. num_fields = len(field_names)
  328. arg_list = repr(field_names).replace("'", "")[1:-1]
  329. repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')'
  330. tuple_new = tuple.__new__
  331. _len = len
  332. # Create all the named tuple methods to be added to the class namespace
  333. s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))'
  334. namespace = {'_tuple_new': tuple_new, '__name__': f'namedtuple_{typename}'}
  335. # Note: exec() has the side-effect of interning the field names
  336. exec(s, namespace)
  337. __new__ = namespace['__new__']
  338. __new__.__doc__ = f'Create new instance of {typename}({arg_list})'
  339. if defaults is not None:
  340. __new__.__defaults__ = defaults
  341. @classmethod
  342. def _make(cls, iterable):
  343. result = tuple_new(cls, iterable)
  344. if _len(result) != num_fields:
  345. raise TypeError(f'Expected {num_fields} arguments, got {len(result)}')
  346. return result
  347. _make.__func__.__doc__ = (f'Make a new {typename} object from a sequence '
  348. 'or iterable')
  349. def _replace(_self, **kwds):
  350. result = _self._make(map(kwds.pop, field_names, _self))
  351. if kwds:
  352. raise ValueError(f'Got unexpected field names: {list(kwds)!r}')
  353. return result
  354. _replace.__doc__ = (f'Return a new {typename} object replacing specified '
  355. 'fields with new values')
  356. def __repr__(self):
  357. 'Return a nicely formatted representation string'
  358. return self.__class__.__name__ + repr_fmt % self
  359. def _asdict(self):
  360. 'Return a new OrderedDict which maps field names to their values.'
  361. return OrderedDict(zip(self._fields, self))
  362. def __getnewargs__(self):
  363. 'Return self as a plain tuple. Used by copy and pickle.'
  364. return tuple(self)
  365. # Modify function metadata to help with introspection and debugging
  366. for method in (__new__, _make.__func__, _replace,
  367. __repr__, _asdict, __getnewargs__):
  368. method.__qualname__ = f'{typename}.{method.__name__}'
  369. # Build-up the class namespace dictionary
  370. # and use type() to build the result class
  371. class_namespace = {
  372. '__doc__': f'{typename}({arg_list})',
  373. '__slots__': (),
  374. '_fields': field_names,
  375. '_field_defaults': field_defaults,
  376. # alternate spelling for backward compatibility
  377. '_fields_defaults': field_defaults,
  378. '__new__': __new__,
  379. '_make': _make,
  380. '_replace': _replace,
  381. '__repr__': __repr__,
  382. '_asdict': _asdict,
  383. '__getnewargs__': __getnewargs__,
  384. }
  385. cache = _nt_itemgetters
  386. for index, name in enumerate(field_names):
  387. try:
  388. itemgetter_object, doc = cache[index]
  389. except KeyError:
  390. itemgetter_object = _itemgetter(index)
  391. doc = f'Alias for field number {index}'
  392. cache[index] = itemgetter_object, doc
  393. class_namespace[name] = property(itemgetter_object, doc=doc)
  394. result = type(typename, (tuple,), class_namespace)
  395. # For pickling to work, the __module__ variable needs to be set to the frame
  396. # where the named tuple is created. Bypass this step in environments where
  397. # sys._getframe is not defined (Jython for example) or sys._getframe is not
  398. # defined for arguments greater than 0 (IronPython), or where the user has
  399. # specified a particular module.
  400. if module is None:
  401. try:
  402. module = _sys._getframe(1).f_globals.get('__name__', '__main__')
  403. except (AttributeError, ValueError):
  404. pass
  405. if module is not None:
  406. result.__module__ = module
  407. return result
  408. ########################################################################
  409. ### Counter
  410. ########################################################################
  411. def _count_elements(mapping, iterable):
  412. 'Tally elements from the iterable.'
  413. mapping_get = mapping.get
  414. for elem in iterable:
  415. mapping[elem] = mapping_get(elem, 0) + 1
  416. try: # Load C helper function if available
  417. from _collections import _count_elements
  418. except ImportError:
  419. pass
  420. class Counter(dict):
  421. '''Dict subclass for counting hashable items. Sometimes called a bag
  422. or multiset. Elements are stored as dictionary keys and their counts
  423. are stored as dictionary values.
  424. >>> c = Counter('abcdeabcdabcaba') # count elements from a string
  425. >>> c.most_common(3) # three most common elements
  426. [('a', 5), ('b', 4), ('c', 3)]
  427. >>> sorted(c) # list all unique elements
  428. ['a', 'b', 'c', 'd', 'e']
  429. >>> ''.join(sorted(c.elements())) # list elements with repetitions
  430. 'aaaaabbbbcccdde'
  431. >>> sum(c.values()) # total of all counts
  432. 15
  433. >>> c['a'] # count of letter 'a'
  434. 5
  435. >>> for elem in 'shazam': # update counts from an iterable
  436. ... c[elem] += 1 # by adding 1 to each element's count
  437. >>> c['a'] # now there are seven 'a'
  438. 7
  439. >>> del c['b'] # remove all 'b'
  440. >>> c['b'] # now there are zero 'b'
  441. 0
  442. >>> d = Counter('simsalabim') # make another counter
  443. >>> c.update(d) # add in the second counter
  444. >>> c['a'] # now there are nine 'a'
  445. 9
  446. >>> c.clear() # empty the counter
  447. >>> c
  448. Counter()
  449. Note: If a count is set to zero or reduced to zero, it will remain
  450. in the counter until the entry is deleted or the counter is cleared:
  451. >>> c = Counter('aaabbc')
  452. >>> c['b'] -= 2 # reduce the count of 'b' by two
  453. >>> c.most_common() # 'b' is still in, but its count is zero
  454. [('a', 3), ('c', 1), ('b', 0)]
  455. '''
  456. # References:
  457. # http://en.wikipedia.org/wiki/Multiset
  458. # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html
  459. # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm
  460. # http://code.activestate.com/recipes/259174/
  461. # Knuth, TAOCP Vol. II section 4.6.3
  462. def __init__(*args, **kwds):
  463. '''Create a new, empty Counter object. And if given, count elements
  464. from an input iterable. Or, initialize the count from another mapping
  465. of elements to their counts.
  466. >>> c = Counter() # a new, empty counter
  467. >>> c = Counter('gallahad') # a new counter from an iterable
  468. >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
  469. >>> c = Counter(a=4, b=2) # a new counter from keyword args
  470. '''
  471. if not args:
  472. raise TypeError("descriptor '__init__' of 'Counter' object "
  473. "needs an argument")
  474. self, *args = args
  475. if len(args) > 1:
  476. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  477. super(Counter, self).__init__()
  478. self.update(*args, **kwds)
  479. def __missing__(self, key):
  480. 'The count of elements not in the Counter is zero.'
  481. # Needed so that self[missing_item] does not raise KeyError
  482. return 0
  483. def most_common(self, n=None):
  484. '''List the n most common elements and their counts from the most
  485. common to the least. If n is None, then list all element counts.
  486. >>> Counter('abcdeabcdabcaba').most_common(3)
  487. [('a', 5), ('b', 4), ('c', 3)]
  488. '''
  489. # Emulate Bag.sortedByCount from Smalltalk
  490. if n is None:
  491. return sorted(self.items(), key=_itemgetter(1), reverse=True)
  492. return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
  493. def elements(self):
  494. '''Iterator over elements repeating each as many times as its count.
  495. >>> c = Counter('ABCABC')
  496. >>> sorted(c.elements())
  497. ['A', 'A', 'B', 'B', 'C', 'C']
  498. # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1
  499. >>> prime_factors = Counter({2: 2, 3: 3, 17: 1})
  500. >>> product = 1
  501. >>> for factor in prime_factors.elements(): # loop over factors
  502. ... product *= factor # and multiply them
  503. >>> product
  504. 1836
  505. Note, if an element's count has been set to zero or is a negative
  506. number, elements() will ignore it.
  507. '''
  508. # Emulate Bag.do from Smalltalk and Multiset.begin from C++.
  509. return _chain.from_iterable(_starmap(_repeat, self.items()))
  510. # Override dict methods where necessary
  511. @classmethod
  512. def fromkeys(cls, iterable, v=None):
  513. # There is no equivalent method for counters because setting v=1
  514. # means that no element can have a count greater than one.
  515. raise NotImplementedError(
  516. 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.')
  517. def update(*args, **kwds):
  518. '''Like dict.update() but add counts instead of replacing them.
  519. Source can be an iterable, a dictionary, or another Counter instance.
  520. >>> c = Counter('which')
  521. >>> c.update('witch') # add elements from another iterable
  522. >>> d = Counter('watch')
  523. >>> c.update(d) # add elements from another counter
  524. >>> c['h'] # four 'h' in which, witch, and watch
  525. 4
  526. '''
  527. # The regular dict.update() operation makes no sense here because the
  528. # replace behavior results in the some of original untouched counts
  529. # being mixed-in with all of the other counts for a mismash that
  530. # doesn't have a straight-forward interpretation in most counting
  531. # contexts. Instead, we implement straight-addition. Both the inputs
  532. # and outputs are allowed to contain zero and negative counts.
  533. if not args:
  534. raise TypeError("descriptor 'update' of 'Counter' object "
  535. "needs an argument")
  536. self, *args = args
  537. if len(args) > 1:
  538. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  539. iterable = args[0] if args else None
  540. if iterable is not None:
  541. if isinstance(iterable, _collections_abc.Mapping):
  542. if self:
  543. self_get = self.get
  544. for elem, count in iterable.items():
  545. self[elem] = count + self_get(elem, 0)
  546. else:
  547. super(Counter, self).update(iterable) # fast path when counter is empty
  548. else:
  549. _count_elements(self, iterable)
  550. if kwds:
  551. self.update(kwds)
  552. def subtract(*args, **kwds):
  553. '''Like dict.update() but subtracts counts instead of replacing them.
  554. Counts can be reduced below zero. Both the inputs and outputs are
  555. allowed to contain zero and negative counts.
  556. Source can be an iterable, a dictionary, or another Counter instance.
  557. >>> c = Counter('which')
  558. >>> c.subtract('witch') # subtract elements from another iterable
  559. >>> c.subtract(Counter('watch')) # subtract elements from another counter
  560. >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch
  561. 0
  562. >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch
  563. -1
  564. '''
  565. if not args:
  566. raise TypeError("descriptor 'subtract' of 'Counter' object "
  567. "needs an argument")
  568. self, *args = args
  569. if len(args) > 1:
  570. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  571. iterable = args[0] if args else None
  572. if iterable is not None:
  573. self_get = self.get
  574. if isinstance(iterable, _collections_abc.Mapping):
  575. for elem, count in iterable.items():
  576. self[elem] = self_get(elem, 0) - count
  577. else:
  578. for elem in iterable:
  579. self[elem] = self_get(elem, 0) - 1
  580. if kwds:
  581. self.subtract(kwds)
  582. def copy(self):
  583. 'Return a shallow copy.'
  584. return self.__class__(self)
  585. def __reduce__(self):
  586. return self.__class__, (dict(self),)
  587. def __delitem__(self, elem):
  588. 'Like dict.__delitem__() but does not raise KeyError for missing values.'
  589. if elem in self:
  590. super().__delitem__(elem)
  591. def __repr__(self):
  592. if not self:
  593. return '%s()' % self.__class__.__name__
  594. try:
  595. items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
  596. return '%s({%s})' % (self.__class__.__name__, items)
  597. except TypeError:
  598. # handle case where values are not orderable
  599. return '{0}({1!r})'.format(self.__class__.__name__, dict(self))
  600. # Multiset-style mathematical operations discussed in:
  601. # Knuth TAOCP Volume II section 4.6.3 exercise 19
  602. # and at http://en.wikipedia.org/wiki/Multiset
  603. #
  604. # Outputs guaranteed to only include positive counts.
  605. #
  606. # To strip negative and zero counts, add-in an empty counter:
  607. # c += Counter()
  608. def __add__(self, other):
  609. '''Add counts from two counters.
  610. >>> Counter('abbb') + Counter('bcc')
  611. Counter({'b': 4, 'c': 2, 'a': 1})
  612. '''
  613. if not isinstance(other, Counter):
  614. return NotImplemented
  615. result = Counter()
  616. for elem, count in self.items():
  617. newcount = count + other[elem]
  618. if newcount > 0:
  619. result[elem] = newcount
  620. for elem, count in other.items():
  621. if elem not in self and count > 0:
  622. result[elem] = count
  623. return result
  624. def __sub__(self, other):
  625. ''' Subtract count, but keep only results with positive counts.
  626. >>> Counter('abbbc') - Counter('bccd')
  627. Counter({'b': 2, 'a': 1})
  628. '''
  629. if not isinstance(other, Counter):
  630. return NotImplemented
  631. result = Counter()
  632. for elem, count in self.items():
  633. newcount = count - other[elem]
  634. if newcount > 0:
  635. result[elem] = newcount
  636. for elem, count in other.items():
  637. if elem not in self and count < 0:
  638. result[elem] = 0 - count
  639. return result
  640. def __or__(self, other):
  641. '''Union is the maximum of value in either of the input counters.
  642. >>> Counter('abbb') | Counter('bcc')
  643. Counter({'b': 3, 'c': 2, 'a': 1})
  644. '''
  645. if not isinstance(other, Counter):
  646. return NotImplemented
  647. result = Counter()
  648. for elem, count in self.items():
  649. other_count = other[elem]
  650. newcount = other_count if count < other_count else count
  651. if newcount > 0:
  652. result[elem] = newcount
  653. for elem, count in other.items():
  654. if elem not in self and count > 0:
  655. result[elem] = count
  656. return result
  657. def __and__(self, other):
  658. ''' Intersection is the minimum of corresponding counts.
  659. >>> Counter('abbb') & Counter('bcc')
  660. Counter({'b': 1})
  661. '''
  662. if not isinstance(other, Counter):
  663. return NotImplemented
  664. result = Counter()
  665. for elem, count in self.items():
  666. other_count = other[elem]
  667. newcount = count if count < other_count else other_count
  668. if newcount > 0:
  669. result[elem] = newcount
  670. return result
  671. def __pos__(self):
  672. 'Adds an empty counter, effectively stripping negative and zero counts'
  673. result = Counter()
  674. for elem, count in self.items():
  675. if count > 0:
  676. result[elem] = count
  677. return result
  678. def __neg__(self):
  679. '''Subtracts from an empty counter. Strips positive and zero counts,
  680. and flips the sign on negative counts.
  681. '''
  682. result = Counter()
  683. for elem, count in self.items():
  684. if count < 0:
  685. result[elem] = 0 - count
  686. return result
  687. def _keep_positive(self):
  688. '''Internal method to strip elements with a negative or zero count'''
  689. nonpositive = [elem for elem, count in self.items() if not count > 0]
  690. for elem in nonpositive:
  691. del self[elem]
  692. return self
  693. def __iadd__(self, other):
  694. '''Inplace add from another counter, keeping only positive counts.
  695. >>> c = Counter('abbb')
  696. >>> c += Counter('bcc')
  697. >>> c
  698. Counter({'b': 4, 'c': 2, 'a': 1})
  699. '''
  700. for elem, count in other.items():
  701. self[elem] += count
  702. return self._keep_positive()
  703. def __isub__(self, other):
  704. '''Inplace subtract counter, but keep only results with positive counts.
  705. >>> c = Counter('abbbc')
  706. >>> c -= Counter('bccd')
  707. >>> c
  708. Counter({'b': 2, 'a': 1})
  709. '''
  710. for elem, count in other.items():
  711. self[elem] -= count
  712. return self._keep_positive()
  713. def __ior__(self, other):
  714. '''Inplace union is the maximum of value from either counter.
  715. >>> c = Counter('abbb')
  716. >>> c |= Counter('bcc')
  717. >>> c
  718. Counter({'b': 3, 'c': 2, 'a': 1})
  719. '''
  720. for elem, other_count in other.items():
  721. count = self[elem]
  722. if other_count > count:
  723. self[elem] = other_count
  724. return self._keep_positive()
  725. def __iand__(self, other):
  726. '''Inplace intersection is the minimum of corresponding counts.
  727. >>> c = Counter('abbb')
  728. >>> c &= Counter('bcc')
  729. >>> c
  730. Counter({'b': 1})
  731. '''
  732. for elem, count in self.items():
  733. other_count = other[elem]
  734. if other_count < count:
  735. self[elem] = other_count
  736. return self._keep_positive()
  737. ########################################################################
  738. ### ChainMap
  739. ########################################################################
  740. class ChainMap(_collections_abc.MutableMapping):
  741. ''' A ChainMap groups multiple dicts (or other mappings) together
  742. to create a single, updateable view.
  743. The underlying mappings are stored in a list. That list is public and can
  744. be accessed or updated using the *maps* attribute. There is no other
  745. state.
  746. Lookups search the underlying mappings successively until a key is found.
  747. In contrast, writes, updates, and deletions only operate on the first
  748. mapping.
  749. '''
  750. def __init__(self, *maps):
  751. '''Initialize a ChainMap by setting *maps* to the given mappings.
  752. If no mappings are provided, a single empty dictionary is used.
  753. '''
  754. self.maps = list(maps) or [{}] # always at least one map
  755. def __missing__(self, key):
  756. raise KeyError(key)
  757. def __getitem__(self, key):
  758. for mapping in self.maps:
  759. try:
  760. return mapping[key] # can't use 'key in mapping' with defaultdict
  761. except KeyError:
  762. pass
  763. return self.__missing__(key) # support subclasses that define __missing__
  764. def get(self, key, default=None):
  765. return self[key] if key in self else default
  766. def __len__(self):
  767. return len(set().union(*self.maps)) # reuses stored hash values if possible
  768. def __iter__(self):
  769. d = {}
  770. for mapping in reversed(self.maps):
  771. d.update(mapping) # reuses stored hash values if possible
  772. return iter(d)
  773. def __contains__(self, key):
  774. return any(key in m for m in self.maps)
  775. def __bool__(self):
  776. return any(self.maps)
  777. @_recursive_repr()
  778. def __repr__(self):
  779. return '{0.__class__.__name__}({1})'.format(
  780. self, ', '.join(map(repr, self.maps)))
  781. @classmethod
  782. def fromkeys(cls, iterable, *args):
  783. 'Create a ChainMap with a single dict created from the iterable.'
  784. return cls(dict.fromkeys(iterable, *args))
  785. def copy(self):
  786. 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
  787. return self.__class__(self.maps[0].copy(), *self.maps[1:])
  788. __copy__ = copy
  789. def new_child(self, m=None): # like Django's Context.push()
  790. '''New ChainMap with a new map followed by all previous maps.
  791. If no map is provided, an empty dict is used.
  792. '''
  793. if m is None:
  794. m = {}
  795. return self.__class__(m, *self.maps)
  796. @property
  797. def parents(self): # like Django's Context.pop()
  798. 'New ChainMap from maps[1:].'
  799. return self.__class__(*self.maps[1:])
  800. def __setitem__(self, key, value):
  801. self.maps[0][key] = value
  802. def __delitem__(self, key):
  803. try:
  804. del self.maps[0][key]
  805. except KeyError:
  806. raise KeyError('Key not found in the first mapping: {!r}'.format(key))
  807. def popitem(self):
  808. 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
  809. try:
  810. return self.maps[0].popitem()
  811. except KeyError:
  812. raise KeyError('No keys found in the first mapping.')
  813. def pop(self, key, *args):
  814. 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
  815. try:
  816. return self.maps[0].pop(key, *args)
  817. except KeyError:
  818. raise KeyError('Key not found in the first mapping: {!r}'.format(key))
  819. def clear(self):
  820. 'Clear maps[0], leaving maps[1:] intact.'
  821. self.maps[0].clear()
  822. ################################################################################
  823. ### UserDict
  824. ################################################################################
  825. class UserDict(_collections_abc.MutableMapping):
  826. # Start by filling-out the abstract methods
  827. def __init__(*args, **kwargs):
  828. if not args:
  829. raise TypeError("descriptor '__init__' of 'UserDict' object "
  830. "needs an argument")
  831. self, *args = args
  832. if len(args) > 1:
  833. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  834. if args:
  835. dict = args[0]
  836. elif 'dict' in kwargs:
  837. dict = kwargs.pop('dict')
  838. import warnings
  839. warnings.warn("Passing 'dict' as keyword argument is deprecated",
  840. DeprecationWarning, stacklevel=2)
  841. else:
  842. dict = None
  843. self.data = {}
  844. if dict is not None:
  845. self.update(dict)
  846. if len(kwargs):
  847. self.update(kwargs)
  848. def __len__(self): return len(self.data)
  849. def __getitem__(self, key):
  850. if key in self.data:
  851. return self.data[key]
  852. if hasattr(self.__class__, "__missing__"):
  853. return self.__class__.__missing__(self, key)
  854. raise KeyError(key)
  855. def __setitem__(self, key, item): self.data[key] = item
  856. def __delitem__(self, key): del self.data[key]
  857. def __iter__(self):
  858. return iter(self.data)
  859. # Modify __contains__ to work correctly when __missing__ is present
  860. def __contains__(self, key):
  861. return key in self.data
  862. # Now, add the methods in dicts but not in MutableMapping
  863. def __repr__(self): return repr(self.data)
  864. def __copy__(self):
  865. inst = self.__class__.__new__(self.__class__)
  866. inst.__dict__.update(self.__dict__)
  867. # Create a copy and avoid triggering descriptors
  868. inst.__dict__["data"] = self.__dict__["data"].copy()
  869. return inst
  870. def copy(self):
  871. if self.__class__ is UserDict:
  872. return UserDict(self.data.copy())
  873. import copy
  874. data = self.data
  875. try:
  876. self.data = {}
  877. c = copy.copy(self)
  878. finally:
  879. self.data = data
  880. c.update(self)
  881. return c
  882. @classmethod
  883. def fromkeys(cls, iterable, value=None):
  884. d = cls()
  885. for key in iterable:
  886. d[key] = value
  887. return d
  888. ################################################################################
  889. ### UserList
  890. ################################################################################
  891. class UserList(_collections_abc.MutableSequence):
  892. """A more or less complete user-defined wrapper around list objects."""
  893. def __init__(self, initlist=None):
  894. self.data = []
  895. if initlist is not None:
  896. # XXX should this accept an arbitrary sequence?
  897. if type(initlist) == type(self.data):
  898. self.data[:] = initlist
  899. elif isinstance(initlist, UserList):
  900. self.data[:] = initlist.data[:]
  901. else:
  902. self.data = list(initlist)
  903. def __repr__(self): return repr(self.data)
  904. def __lt__(self, other): return self.data < self.__cast(other)
  905. def __le__(self, other): return self.data <= self.__cast(other)
  906. def __eq__(self, other): return self.data == self.__cast(other)
  907. def __gt__(self, other): return self.data > self.__cast(other)
  908. def __ge__(self, other): return self.data >= self.__cast(other)
  909. def __cast(self, other):
  910. return other.data if isinstance(other, UserList) else other
  911. def __contains__(self, item): return item in self.data
  912. def __len__(self): return len(self.data)
  913. def __getitem__(self, i):
  914. if isinstance(i, slice):
  915. return self.__class__(self.data[i])
  916. else:
  917. return self.data[i]
  918. def __setitem__(self, i, item): self.data[i] = item
  919. def __delitem__(self, i): del self.data[i]
  920. def __add__(self, other):
  921. if isinstance(other, UserList):
  922. return self.__class__(self.data + other.data)
  923. elif isinstance(other, type(self.data)):
  924. return self.__class__(self.data + other)
  925. return self.__class__(self.data + list(other))
  926. def __radd__(self, other):
  927. if isinstance(other, UserList):
  928. return self.__class__(other.data + self.data)
  929. elif isinstance(other, type(self.data)):
  930. return self.__class__(other + self.data)
  931. return self.__class__(list(other) + self.data)
  932. def __iadd__(self, other):
  933. if isinstance(other, UserList):
  934. self.data += other.data
  935. elif isinstance(other, type(self.data)):
  936. self.data += other
  937. else:
  938. self.data += list(other)
  939. return self
  940. def __mul__(self, n):
  941. return self.__class__(self.data*n)
  942. __rmul__ = __mul__
  943. def __imul__(self, n):
  944. self.data *= n
  945. return self
  946. def __copy__(self):
  947. inst = self.__class__.__new__(self.__class__)
  948. inst.__dict__.update(self.__dict__)
  949. # Create a copy and avoid triggering descriptors
  950. inst.__dict__["data"] = self.__dict__["data"][:]
  951. return inst
  952. def append(self, item): self.data.append(item)
  953. def insert(self, i, item): self.data.insert(i, item)
  954. def pop(self, i=-1): return self.data.pop(i)
  955. def remove(self, item): self.data.remove(item)
  956. def clear(self): self.data.clear()
  957. def copy(self): return self.__class__(self)
  958. def count(self, item): return self.data.count(item)
  959. def index(self, item, *args): return self.data.index(item, *args)
  960. def reverse(self): self.data.reverse()
  961. def sort(self, *args, **kwds): self.data.sort(*args, **kwds)
  962. def extend(self, other):
  963. if isinstance(other, UserList):
  964. self.data.extend(other.data)
  965. else:
  966. self.data.extend(other)
  967. ################################################################################
  968. ### UserString
  969. ################################################################################
  970. class UserString(_collections_abc.Sequence):
  971. def __init__(self, seq):
  972. if isinstance(seq, str):
  973. self.data = seq
  974. elif isinstance(seq, UserString):
  975. self.data = seq.data[:]
  976. else:
  977. self.data = str(seq)
  978. def __str__(self): return str(self.data)
  979. def __repr__(self): return repr(self.data)
  980. def __int__(self): return int(self.data)
  981. def __float__(self): return float(self.data)
  982. def __complex__(self): return complex(self.data)
  983. def __hash__(self): return hash(self.data)
  984. def __getnewargs__(self):
  985. return (self.data[:],)
  986. def __eq__(self, string):
  987. if isinstance(string, UserString):
  988. return self.data == string.data
  989. return self.data == string
  990. def __lt__(self, string):
  991. if isinstance(string, UserString):
  992. return self.data < string.data
  993. return self.data < string
  994. def __le__(self, string):
  995. if isinstance(string, UserString):
  996. return self.data <= string.data
  997. return self.data <= string
  998. def __gt__(self, string):
  999. if isinstance(string, UserString):
  1000. return self.data > string.data
  1001. return self.data > string
  1002. def __ge__(self, string):
  1003. if isinstance(string, UserString):
  1004. return self.data >= string.data
  1005. return self.data >= string
  1006. def __contains__(self, char):
  1007. if isinstance(char, UserString):
  1008. char = char.data
  1009. return char in self.data
  1010. def __len__(self): return len(self.data)
  1011. def __getitem__(self, index): return self.__class__(self.data[index])
  1012. def __add__(self, other):
  1013. if isinstance(other, UserString):
  1014. return self.__class__(self.data + other.data)
  1015. elif isinstance(other, str):
  1016. return self.__class__(self.data + other)
  1017. return self.__class__(self.data + str(other))
  1018. def __radd__(self, other):
  1019. if isinstance(other, str):
  1020. return self.__class__(other + self.data)
  1021. return self.__class__(str(other) + self.data)
  1022. def __mul__(self, n):
  1023. return self.__class__(self.data*n)
  1024. __rmul__ = __mul__
  1025. def __mod__(self, args):
  1026. return self.__class__(self.data % args)
  1027. def __rmod__(self, format):
  1028. return self.__class__(format % args)
  1029. # the following methods are defined in alphabetical order:
  1030. def capitalize(self): return self.__class__(self.data.capitalize())
  1031. def casefold(self):
  1032. return self.__class__(self.data.casefold())
  1033. def center(self, width, *args):
  1034. return self.__class__(self.data.center(width, *args))
  1035. def count(self, sub, start=0, end=_sys.maxsize):
  1036. if isinstance(sub, UserString):
  1037. sub = sub.data
  1038. return self.data.count(sub, start, end)
  1039. def encode(self, encoding=None, errors=None): # XXX improve this?
  1040. if encoding:
  1041. if errors:
  1042. return self.__class__(self.data.encode(encoding, errors))
  1043. return self.__class__(self.data.encode(encoding))
  1044. return self.__class__(self.data.encode())
  1045. def endswith(self, suffix, start=0, end=_sys.maxsize):
  1046. return self.data.endswith(suffix, start, end)
  1047. def expandtabs(self, tabsize=8):
  1048. return self.__class__(self.data.expandtabs(tabsize))
  1049. def find(self, sub, start=0, end=_sys.maxsize):
  1050. if isinstance(sub, UserString):
  1051. sub = sub.data
  1052. return self.data.find(sub, start, end)
  1053. def format(self, *args, **kwds):
  1054. return self.data.format(*args, **kwds)
  1055. def format_map(self, mapping):
  1056. return self.data.format_map(mapping)
  1057. def index(self, sub, start=0, end=_sys.maxsize):
  1058. return self.data.index(sub, start, end)
  1059. def isalpha(self): return self.data.isalpha()
  1060. def isalnum(self): return self.data.isalnum()
  1061. def isascii(self): return self.data.isascii()
  1062. def isdecimal(self): return self.data.isdecimal()
  1063. def isdigit(self): return self.data.isdigit()
  1064. def isidentifier(self): return self.data.isidentifier()
  1065. def islower(self): return self.data.islower()
  1066. def isnumeric(self): return self.data.isnumeric()
  1067. def isprintable(self): return self.data.isprintable()
  1068. def isspace(self): return self.data.isspace()
  1069. def istitle(self): return self.data.istitle()
  1070. def isupper(self): return self.data.isupper()
  1071. def join(self, seq): return self.data.join(seq)
  1072. def ljust(self, width, *args):
  1073. return self.__class__(self.data.ljust(width, *args))
  1074. def lower(self): return self.__class__(self.data.lower())
  1075. def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars))
  1076. maketrans = str.maketrans
  1077. def partition(self, sep):
  1078. return self.data.partition(sep)
  1079. def replace(self, old, new, maxsplit=-1):
  1080. if isinstance(old, UserString):
  1081. old = old.data
  1082. if isinstance(new, UserString):
  1083. new = new.data
  1084. return self.__class__(self.data.replace(old, new, maxsplit))
  1085. def rfind(self, sub, start=0, end=_sys.maxsize):
  1086. if isinstance(sub, UserString):
  1087. sub = sub.data
  1088. return self.data.rfind(sub, start, end)
  1089. def rindex(self, sub, start=0, end=_sys.maxsize):
  1090. return self.data.rindex(sub, start, end)
  1091. def rjust(self, width, *args):
  1092. return self.__class__(self.data.rjust(width, *args))
  1093. def rpartition(self, sep):
  1094. return self.data.rpartition(sep)
  1095. def rstrip(self, chars=None):
  1096. return self.__class__(self.data.rstrip(chars))
  1097. def split(self, sep=None, maxsplit=-1):
  1098. return self.data.split(sep, maxsplit)
  1099. def rsplit(self, sep=None, maxsplit=-1):
  1100. return self.data.rsplit(sep, maxsplit)
  1101. def splitlines(self, keepends=False): return self.data.splitlines(keepends)
  1102. def startswith(self, prefix, start=0, end=_sys.maxsize):
  1103. return self.data.startswith(prefix, start, end)
  1104. def strip(self, chars=None): return self.__class__(self.data.strip(chars))
  1105. def swapcase(self): return self.__class__(self.data.swapcase())
  1106. def title(self): return self.__class__(self.data.title())
  1107. def translate(self, *args):
  1108. return self.__class__(self.data.translate(*args))
  1109. def upper(self): return self.__class__(self.data.upper())
  1110. def zfill(self, width): return self.__class__(self.data.zfill(width))