From 2b5cc117c616d81590a060fcbffe314e00604835 Mon Sep 17 00:00:00 2001 From: James Cave Date: Sat, 22 Jul 2023 21:58:09 -0400 Subject: [PATCH 1/4] gh-107089: Improve Shelf.clear method performance The clear method used to be implemented by inheriting a mix-in from the MutableMapping ABC. It was a poor fit for shelves, and a better implementation is now in place --- Lib/shelve.py | 19 +++++++++++++++++++ Misc/ACKS | 1 + ...-07-22-21-57-34.gh-issue-107089.Dnget2.rst | 2 ++ 3 files changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst diff --git a/Lib/shelve.py b/Lib/shelve.py index e053c397345a07..fd908bc23fe0ad 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -171,6 +171,25 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() + def clear(self): + """Remove all items from the shelf.""" + self.cache.clear() + try: + self.dict.clear() + except AttributeError: + # dbm objects don't have a clear method, so we need to do + # the clearing here. + keys = self.dict.keys() + if not isinstance(keys, list): + # The keys method on dbm objects returns a list. + # Typically, the keys method on a mapping returns a + # lazy iterator, so we need to watch out for that in + # case someone passes in a backing object that behaves + # that way. + keys = list(keys) + for k in keys: + del self.dict[k] + class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface. diff --git a/Misc/ACKS b/Misc/ACKS index fadf488888aa8b..8b8c5ad8434bd7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -289,6 +289,7 @@ Edward Catmur Lorenzo M. Catucci Bruno Cauet Donn Cave +James Cave Charles Cazabon Jesús Cea Avión Per Cederqvist diff --git a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst new file mode 100644 index 00000000000000..c67301a918ae60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst @@ -0,0 +1,2 @@ +The :meth:`Shelf.clear` method in the :mod:`shelve` module is much faster. +Patch by James Cave. From 0e3f387ca1f5c25b0b59dbb81064bd1e4b57df55 Mon Sep 17 00:00:00 2001 From: James Cave Date: Mon, 24 Jul 2023 10:41:29 -0400 Subject: [PATCH 2/4] gh-107089: call dbm.clear method Because gh-107089 is peculiar to implementation details of dbm objects, it would be less disruptive to implement it in the DbfilenameShelf class, which is used for calls to shelve.open. Since it is known that the backing object is specifically one of the dbm objects, its clear method (see gh-107122) can be used with no fallback code. --- Lib/shelve.py | 27 +++++-------------- ...-07-22-21-57-34.gh-issue-107089.Dnget2.rst | 4 +-- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index fd908bc23fe0ad..d71b243dd359e2 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -171,26 +171,6 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() - def clear(self): - """Remove all items from the shelf.""" - self.cache.clear() - try: - self.dict.clear() - except AttributeError: - # dbm objects don't have a clear method, so we need to do - # the clearing here. - keys = self.dict.keys() - if not isinstance(keys, list): - # The keys method on dbm objects returns a list. - # Typically, the keys method on a mapping returns a - # lazy iterator, so we need to watch out for that in - # case someone passes in a backing object that behaves - # that way. - keys = list(keys) - for k in keys: - del self.dict[k] - - class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface. @@ -244,6 +224,13 @@ class DbfilenameShelf(Shelf): def __init__(self, filename, flag='c', protocol=None, writeback=False): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + + def clear(self): + """Remove all items from the shelf.""" + # Call through to the clear method on dbm-backed shelves. + # see https://github.com/python/cpython/issues/107089 + self.cache.clear() + self.dict.clear() def open(filename, flag='c', protocol=None, writeback=False): diff --git a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst index c67301a918ae60..9d5ba1a2d7ccba 100644 --- a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst +++ b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst @@ -1,2 +1,2 @@ -The :meth:`Shelf.clear` method in the :mod:`shelve` module is much faster. -Patch by James Cave. +Shelves opened with :func:`shelve.open` have a much faster :meth:`clear` +method. Patch by James Cave. \ No newline at end of file From 0f413694df2291ba7a7a769713753c72828c8113 Mon Sep 17 00:00:00 2001 From: James Cave Date: Mon, 24 Jul 2023 10:56:09 -0400 Subject: [PATCH 3/4] fix whitespace --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index d71b243dd359e2..ec6cf3073c0955 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -224,7 +224,7 @@ class DbfilenameShelf(Shelf): def __init__(self, filename, flag='c', protocol=None, writeback=False): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) - + def clear(self): """Remove all items from the shelf.""" # Call through to the clear method on dbm-backed shelves. From d90b207029f30c4be77f4d31cb0ca116fe99bd84 Mon Sep 17 00:00:00 2001 From: James Cave Date: Thu, 27 Jul 2023 16:48:26 -0400 Subject: [PATCH 4/4] fix whitespace --- Lib/shelve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/shelve.py b/Lib/shelve.py index ec6cf3073c0955..50584716e9ea64 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -171,6 +171,7 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() + class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface.