Merge ~cjwatson/launchpad:py3-pgsession-datetime-compatibility into launchpad:master

Proposed by Colin Watson
Status: Merged
Approved by: Colin Watson
Approved revision: ccdce08a40faa964891a75adcde8bc036b82d258
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~cjwatson/launchpad:py3-pgsession-datetime-compatibility
Merge into: launchpad:master
Diff against target: 116 lines (+74/-1)
2 files modified
lib/lp/services/webapp/pgsession.py (+33/-1)
lib/lp/services/webapp/tests/test_pgsession.py (+41/-0)
Reviewer Review Type Date Requested Status
Cristian Gonzalez (community) Approve
Review via email: mp+399133@code.launchpad.net

Commit message

Handle unpickling of Python 2 datetime objects on Python 3

Description of the change

Some of the pickled objects in the session database are `datetime.datetime` objects (`logintime` and `last_write`). There are particular difficulties with unpickling these objects on various Python 3 versions, as described in https://bugs.python.org/issue22005. Work around these using a customized unpickler.

To post a comment you must log in.
Revision history for this message
Cristian Gonzalez (cristiangsp) wrote :

Looks good and well tested!

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lib/lp/services/webapp/pgsession.py b/lib/lp/services/webapp/pgsession.py
2index 3f035d3..0a5c664 100644
3--- a/lib/lp/services/webapp/pgsession.py
4+++ b/lib/lp/services/webapp/pgsession.py
5@@ -6,7 +6,9 @@
6 __metaclass__ = type
7
8 from collections import MutableMapping
9+from datetime import datetime
10 import hashlib
11+import io
12 import time
13
14 from lazr.restful.utils import get_current_browser_request
15@@ -30,6 +32,35 @@ HOURS = 60 * MINUTES
16 DAYS = 24 * HOURS
17
18
19+if six.PY3:
20+ class Python2FriendlyUnpickler(pickle._Unpickler):
21+ """An unpickler that handles Python 2 datetime objects.
22+
23+ Python 3 versions before 3.6 fail to unpickle Python 2 datetime
24+ objects (https://bugs.python.org/issue22005); even in Python >= 3.6
25+ they require passing a different encoding to pickle.loads, which may
26+ have undesirable effects on other objects being unpickled. Work
27+ around this by instead patching in a different encoding just for the
28+ argument to datetime.datetime.
29+ """
30+
31+ def find_class(self, module, name):
32+ if module == 'datetime' and name == 'datetime':
33+ original_encoding = self.encoding
34+ self.encoding = 'bytes'
35+
36+ def datetime_factory(pickle_data):
37+ self.encoding = original_encoding
38+ return datetime(pickle_data)
39+
40+ return datetime_factory
41+ else:
42+ return super(Python2FriendlyUnpickler, self).find_class(
43+ module, name)
44+else:
45+ Python2FriendlyUnpickler = pickle.Unpickler
46+
47+
48 class PGSessionBase:
49 store_name = 'session'
50
51@@ -186,7 +217,8 @@ class PGSessionPkgData(MutableMapping, PGSessionBase):
52 result = self.store.execute(
53 query, (self.session_data.hashed_client_id, self.product_id))
54 for key, pickled_value in result:
55- value = pickle.loads(bytes(pickled_value))
56+ value = Python2FriendlyUnpickler(
57+ io.BytesIO(bytes(pickled_value))).load()
58 self._data_cache[key] = value
59
60 def __getitem__(self, key):
61diff --git a/lib/lp/services/webapp/tests/test_pgsession.py b/lib/lp/services/webapp/tests/test_pgsession.py
62index 971ad4c..4d033fd 100644
63--- a/lib/lp/services/webapp/tests/test_pgsession.py
64+++ b/lib/lp/services/webapp/tests/test_pgsession.py
65@@ -5,6 +5,7 @@
66
67 __metaclass__ = type
68
69+from datetime import datetime
70 import hashlib
71
72 from zope.publisher.browser import TestRequest
73@@ -167,3 +168,43 @@ class TestPgSession(TestCase):
74
75 # also see the page test xx-no-anonymous-session-cookies for tests of
76 # the cookie behaviour.
77+
78+ def test_datetime_compatibility(self):
79+ # datetime objects serialized by either Python 2 or 3 can be
80+ # unserialized as part of the session.
81+ client_id = u'Client Id #1'
82+ product_id = u'Product Id'
83+ expected_datetime = datetime(2021, 3, 4, 0, 50, 1, 300000)
84+
85+ session = self.sdc[client_id]
86+ session._ensureClientId()
87+
88+ # These are returned by the following code in Python 2.7 and 3.5
89+ # respectively:
90+ #
91+ # pickle.dumps(expected_datetime, protocol=2)
92+ python_2_pickle = (
93+ b'\x80\x02cdatetime\ndatetime\nq\x00'
94+ b'U\n\x07\xe5\x03\x04\x002\x01\x04\x93\xe0q\x01\x85q\x02Rq\x03.')
95+ python_3_pickle = (
96+ b'\x80\x02cdatetime\ndatetime\nq\x00'
97+ b'c_codecs\nencode\nq\x01'
98+ b'X\r\x00\x00\x00\x07\xc3\xa5\x03\x04\x002\x01\x04\xc2\x93\xc3\xa0'
99+ b'q\x02X\x06\x00\x00\x00latin1q\x03\x86q\x04Rq\x05\x85q\x06R'
100+ b'q\x07.')
101+
102+ store = self.sdc.store
103+ store.execute(
104+ "SELECT set_session_pkg_data(?, ?, ?, ?)",
105+ (session.hashed_client_id, product_id, u'logintime',
106+ python_2_pickle),
107+ noresult=True)
108+ store.execute(
109+ "SELECT set_session_pkg_data(?, ?, ?, ?)",
110+ (session.hashed_client_id, product_id, u'last_write',
111+ python_3_pickle),
112+ noresult=True)
113+
114+ pkgdata = session[product_id]
115+ self.assertEqual(expected_datetime, pkgdata['logintime'])
116+ self.assertEqual(expected_datetime, pkgdata['last_write'])

Subscribers

People subscribed via source and target branches

to status/vote changes: