Merge lp:~cjwatson/storm/py3-dict into lp:storm

[Colin Watson]

Handle Python 3's changes to dict iteration methods.

In most cases I opted for a somewhat pedantic translation of d.iterfoo() to six.iterfoo(d) and d.foo() to list(six.iterfoo(d)), since there are enough cases where this matters either for performance or to avoid iterating over a dictionary while modifying it that it's simplest to be consistent.

There were a few cases where the Python 2 code was iterating over something like d.items() when it didn't need to do a copy; I left those cases alone, since they'll just become slightly more efficient under Python 3.

[Kristian Glass (doismellburning)]

review: Approve

=== modified file 'storm/cache.py'
--- storm/cache.py	2019-06-05 11:41:07 +0000
+++ storm/cache.py	2019-08-11 09:53:43 +0000
@@ -2,6 +2,8 @@
 
 import itertools
 
+import six
+
 
 class Cache(object):
     """Prevents recently used objects from being deallocated.
@@ -142,9 +144,10 @@
         objects, but no more than twice that number.
         """
         self._size = size
-        cache = itertools.islice(itertools.chain(self._new_cache.iteritems(),
-                                                 self._old_cache.iteritems()),
-                                 0, size)
+        cache = itertools.islice(
+            itertools.chain(six.iteritems(self._new_cache),
+                            six.iteritems(self._old_cache)),
+            0, size)
         self._new_cache = dict(cache)
         self._old_cache.clear()
 
 
=== modified file 'storm/cextensions.c'
--- storm/cextensions.c	2019-06-19 09:34:36 +0000
+++ storm/cextensions.c	2019-08-11 09:53:43 +0000
@@ -1931,7 +1931,7 @@
     PyObject *column, *variable, *tmp;
     Py_ssize_t i = 0;
 
-    /* for variable in self.variables.itervalues(): */
+    /* for variable in six.itervalues(self.variables): */
     while (PyDict_Next(self->variables, &i, &column, &variable)) {
         /* variable.checkpoint() */
         CATCH(NULL, tmp = PyObject_CallMethod(variable, "checkpoint", NULL));
 
=== modified file 'storm/expr.py'
--- storm/expr.py	2019-06-07 16:36:44 +0000
+++ storm/expr.py	2019-08-11 09:53:43 +0000
@@ -740,7 +740,7 @@
     state.context = EXPR
     values = insert.values
     if values is Undef:
-        values = [tuple(insert.map.itervalues())]
+        values = [tuple(six.itervalues(insert.map))]
     if isinstance(values, Expr):
         compiled_values = compile(values, state)
     else:
 
=== modified file 'storm/info.py'
--- storm/info.py	2019-06-05 11:41:07 +0000
+++ storm/info.py	2019-08-11 09:53:43 +0000
@@ -22,6 +22,8 @@
 
 from weakref import ref
 
+import six
+
 from storm.exceptions import ClassInfoError
 from storm.expr import Column, Desc, TABLE
 from storm.expr import compile, Table
@@ -194,7 +196,7 @@
         self.event.emit("object-deleted")
 
     def checkpoint(self):
-        for variable in self.variables.itervalues():
+        for variable in six.itervalues(self.variables):
             variable.checkpoint()
 
 
 
=== modified file 'storm/references.py'
--- storm/references.py	2019-06-05 11:41:07 +0000
+++ storm/references.py	2019-08-11 09:53:43 +0000
@@ -22,6 +22,8 @@
 
 import weakref
 
+import six
+
 from storm.exceptions import (
     ClassInfoError, FeatureError, NoStoreError, WrongStoreError)
 from storm.store import Store, get_where_for_args, LostObjectError
@@ -940,14 +942,14 @@
 
 def _find_descriptor_class(used_cls, descr):
     for cls in used_cls.__mro__:
-        for attr, _descr in cls.__dict__.iteritems():
+        for attr, _descr in six.iteritems(cls.__dict__):
             if _descr is descr:
                 return cls
     raise RuntimeError("Reference used in an unknown class")
 
 def _find_descriptor_obj(used_cls, descr):
     for cls in used_cls.__mro__:
-        for attr, _descr in cls.__dict__.iteritems():
+        for attr, _descr in six.iteritems(cls.__dict__):
             if _descr is descr:
                 return getattr(cls, attr)
     raise RuntimeError("Reference used in an unknown class")
 
=== modified file 'storm/sqlobject.py'
--- storm/sqlobject.py	2019-06-05 11:41:07 +0000
+++ storm/sqlobject.py	2019-08-11 09:53:43 +0000
@@ -28,6 +28,8 @@
 import re
 import warnings
 
+import six
+
 from storm.properties import (
     RawStr, Int, Bool, Float, DateTime, Date, TimeDelta)
 from storm.references import Reference, ReferenceSet
@@ -164,7 +166,7 @@
         dict["__storm_table__"] = table_name
 
         attr_to_prop = {}
-        for attr, prop in dict.items():
+        for attr, prop in list(six.iteritems(dict)):
             attr_to_prop[attr] = attr
             if isinstance(prop, ForeignKey):
                 db_name = prop.kwargs.get("dbName", attr)
@@ -224,7 +226,7 @@
         property_registry.add_property(obj, getattr(obj, "id"),
                                        "<table %s>" % table_name)
 
-        for fake_name, real_name in attr_to_prop.items():
+        for fake_name, real_name in list(six.iteritems(attr_to_prop)):
             prop = getattr(obj, real_name)
             if fake_name != real_name:
                 property_registry.add_property(obj, prop, fake_name)
@@ -298,7 +300,7 @@
         self._init(None)
 
     def set(self, **kwargs):
-        for attr, value in kwargs.iteritems():
+        for attr, value in six.iteritems(kwargs):
             setattr(self, attr, value)
 
     def destroySelf(self):
@@ -406,7 +408,7 @@
 
     def _copy(self, **kwargs):
         copy = self.__class__(self._cls, **kwargs)
-        for name, value in self.__dict__.iteritems():
+        for name, value in six.iteritems(self.__dict__):
             if name[1:] not in kwargs and name != "_finished_result_set":
                 setattr(copy, name, value)
         return copy
 
=== modified file 'storm/store.py'
--- storm/store.py	2019-06-05 11:41:07 +0000
+++ storm/store.py	2019-08-11 09:53:43 +0000
@@ -474,7 +474,7 @@
         self._dirty = flushing
 
         predecessors = {}
-        for (before_info, after_info), n in self._order.iteritems():
+        for (before_info, after_info), n in six.iteritems(self._order):
             if n > 0:
                 before_set = predecessors.get(after_info)
                 if before_set is None:
@@ -853,7 +853,7 @@
             del obj_info["primary_vars"]
 
     def _iter_alive(self):
-        return self._alive.values()
+        return list(six.itervalues(self._alive))
 
     def _enable_change_notification(self, obj_info):
         obj_info.event.emit("start-tracking-changes", self._event)
@@ -1406,7 +1406,7 @@
                     for column in changes:
                         obj_info.variables[column].set(AutoReload)
         else:
-            changes = changes.items()
+            changes = list(six.iteritems(changes))
             for obj in cached:
                 for column, value in changes:
                     variables = get_obj_info(obj).variables
 
=== modified file 'storm/tz.py'
--- storm/tz.py	2019-08-09 13:09:09 +0000
+++ storm/tz.py	2019-08-11 09:53:43 +0000
@@ -15,6 +15,8 @@
 import sys
 import os
 
+import six
+
 relativedelta = None
 parser = None
 rrule = None
@@ -714,7 +716,7 @@
 
     def get(self, tzid=None):
         if tzid is None:
-            keys = self._vtz.keys()
+            keys = list(six.iterkeys(self._vtz))
             if len(keys) == 0:
                 raise Exception("no timezones defined")
             elif len(keys) > 1:
 
=== modified file 'storm/uri.py'
--- storm/uri.py	2019-06-07 17:14:33 +0000
+++ storm/uri.py	2019-08-11 09:53:43 +0000
@@ -20,6 +20,7 @@
 #
 from __future__ import print_function
 
+import six
 from six.moves.urllib.parse import quote
 
 from storm.exceptions import URIError
@@ -104,7 +105,7 @@
             append(escape(self.database, "/"))
         if self.options:
             options = ["%s=%s" % (escape(key), escape(value))
-                       for key, value in sorted(self.options.iteritems())]
+                       for key, value in sorted(six.iteritems(self.options))]
             append("?")
             append("&".join(options))
         return "".join(tokens)
 
=== modified file 'storm/zope/testing.py'
--- storm/zope/testing.py	2019-06-05 11:41:07 +0000
+++ storm/zope/testing.py	2019-08-11 09:53:43 +0000
@@ -23,6 +23,7 @@
 import os
 import shutil
 
+import six
 import transaction
 
 from testresources import TestResourceManager
@@ -99,8 +100,9 @@
             # Adapt the old databases format to the new one, for backward
             # compatibility. This should be eventually dropped.
             if isinstance(databases, dict):
-                databases = [{"name": name, "uri": uri, "schema": schema}
-                             for name, (uri, schema) in databases.iteritems()]
+                databases = [
+                    {"name": name, "uri": uri, "schema": schema}
+                    for name, (uri, schema) in six.iteritems(databases)]
 
             # Provide the global IZStorm utility before applying patches, so
             # patch code can get the ztorm object if needed (e.g. looking up
 
=== modified file 'storm/zope/zstorm.py'
--- storm/zope/zstorm.py	2019-06-06 10:14:55 +0000
+++ storm/zope/zstorm.py	2019-08-11 09:53:43 +0000
@@ -31,6 +31,7 @@
 
 from uuid import uuid4
 
+import six
 from zope.interface import implementer
 
 import transaction
@@ -206,11 +207,11 @@
 
     def iterstores(self):
         """Iterate C{name, store} 2-tuples."""
-        # items is explicitly used here, instead of iteritems, to
-        # avoid the problem where a store is deallocated during
+        # We explicitly copy the list of items before iterating over
+        # it to avoid the problem where a store is deallocated during
         # iteration causing RuntimeError: dictionary changed size
         # during iteration.
-        for store, name in self._name_index.items():
+        for store, name in list(six.iteritems(self._name_index)):
             yield name, store
 
     def get_name(self, store):
 
=== modified file 'tests/mocker.py'
--- tests/mocker.py	2019-06-07 17:14:33 +0000
+++ tests/mocker.py	2019-08-11 09:53:43 +0000
@@ -13,6 +13,7 @@
 import os
 import gc
 
+import six
 from six.moves import builtins
 
 
@@ -1109,13 +1110,13 @@
         frame = sys._getframe(depth+1)
     except:
         return None
-    for name, frame_obj in frame.f_locals.iteritems():
+    for name, frame_obj in six.iteritems(frame.f_locals):
         if frame_obj is obj:
             return name
     self = frame.f_locals.get("self")
     if self is not None:
         try:
-            items = list(self.__dict__.iteritems())
+            items = list(six.iteritems(self.__dict__))
         except:
             pass
         else:
@@ -1264,7 +1265,7 @@
                 result = "del %s.%s" % (result, action.args[0])
             elif action.kind == "call":
                 args = [repr(x) for x in action.args]
-                items = list(action.kwargs.iteritems())
+                items = list(six.iteritems(action.kwargs))
                 items.sort()
                 for pair in items:
                     args.append("%s=%r" % pair)
@@ -1384,7 +1385,7 @@
 
     # Either we have the same number of kwargs, or unknown keywords are
     # accepted (KWARGS was used), so check just the ones in kwargs1.
-    for key, arg1 in kwargs1.iteritems():
+    for key, arg1 in six.iteritems(kwargs1):
         if key not in kwargs2:
             return False
         arg2 = kwargs2[key]
@@ -1914,7 +1915,7 @@
     for referrer in gc.get_referrers(remove):
         if (type(referrer) is dict and
             referrer.get("__mocker_replace__", True)):
-            for key, value in referrer.items():
+            for key, value in list(six.iteritems(referrer)):
                 if value is remove:
                     referrer[key] = install
 
@@ -1986,7 +1987,7 @@
         for kind in self._monitored:
             attr = self._get_kind_attr(kind)
             seen = set()
-            for obj in self._monitored[kind].itervalues():
+            for obj in six.itervalues(self._monitored[kind]):
                 cls = type(obj)
                 if issubclass(cls, type):
                     cls = obj
@@ -2000,7 +2001,7 @@
                                     self.execute)
 
     def restore(self):
-        for obj, attr, original in self._patched.itervalues():
+        for obj, attr, original in six.itervalues(self._patched):
             if original is Undefined:
                 delattr(obj, attr)
             else:
 
=== modified file 'tests/store/base.py'
--- tests/store/base.py	2019-06-07 17:14:33 +0000
+++ tests/store/base.py	2019-08-11 09:53:43 +0000
@@ -4946,7 +4946,7 @@
         foo = self.store.get(DictFoo, 20)
         foo["a"] = 1
 
-        self.assertEquals(foo.items(), [("a", 1)])
+        self.assertEquals(list(six.iteritems(foo)), [("a", 1)])
 
         new_obj = DictFoo()
         new_obj.id = 40
 
=== modified file 'tests/zope/testing.py'
--- tests/zope/testing.py	2019-06-05 11:41:07 +0000
+++ tests/zope/testing.py	2019-08-11 09:53:43 +0000
@@ -23,6 +23,8 @@
 import os
 import sys
 
+import six
+
 from tests.helper import TestHelper
 from tests.zope import has_transaction, has_zope_component, has_testresources
 
@@ -239,7 +241,7 @@
         real_invalidate = store.invalidate
 
         def invalidate_proxy():
-            self.assertEqual(0, len(store._alive.values()))
+            self.assertEqual(0, len(list(six.itervalues(store._alive))))
             real_invalidate()
         store.invalidate = invalidate_proxy