Merge lp:~wgrant/launchpad/bug-1642411 into lp:launchpad

Proposed by William Grant on 2016-11-16
Status: Merged
Merged at revision: 18274
Proposed branch: lp:~wgrant/launchpad/bug-1642411
Merge into: lp:launchpad
Diff against target: 107 lines (+61/-3)
2 files modified
lib/lp/services/librarianserver/librariangc.py (+1/-1)
lib/lp/services/librarianserver/tests/test_gc.py (+60/-2)
To merge this branch: bzr merge lp:~wgrant/launchpad/bug-1642411
Reviewer Review Type Date Requested Status
Colin Watson 2016-11-16 Approve on 2016-11-16
Review via email: mp+311092@code.launchpad.net

Commit message

Fix delete_unwanted_swift_files to not crash on segments.

Description of the change

Fix delete_unwanted_swift_files to not crash on segments.

To post a comment you must log in.
Colin Watson (cjwatson) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/services/librarianserver/librariangc.py'
2--- lib/lp/services/librarianserver/librariangc.py 2016-06-20 07:18:17 +0000
3+++ lib/lp/services/librarianserver/librariangc.py 2016-11-16 23:20:45 +0000
4@@ -725,7 +725,7 @@
5 names = sorted(
6 swift_connection.get_container(
7 container, full_listing=True)[1],
8- key=lambda x: int(x['name']))
9+ key=lambda x: map(int, x['name'].split('/')))
10 for name in names:
11 yield (container, name)
12 except swiftclient.ClientException as x:
13
14=== modified file 'lib/lp/services/librarianserver/tests/test_gc.py'
15--- lib/lp/services/librarianserver/tests/test_gc.py 2016-06-20 07:18:17 +0000
16+++ lib/lp/services/librarianserver/tests/test_gc.py 2016-11-16 23:20:45 +0000
17@@ -21,6 +21,7 @@
18
19 from contextlib import contextmanager
20 from sqlobject import SQLObjectNotFound
21+from storm.store import Store
22 from swiftclient import client as swiftclient
23 import transaction
24
25@@ -43,7 +44,10 @@
26 from lp.services.log.logger import BufferLogger
27 from lp.services.features.testing import FeatureFixture
28 from lp.services.utils import utc_now
29-from lp.testing import TestCase
30+from lp.testing import (
31+ monkey_patch,
32+ TestCase,
33+ )
34 from lp.testing.dbuser import switch_dbuser
35 from lp.testing.layers import (
36 LaunchpadZopelessLayer,
37@@ -723,8 +727,10 @@
38 swift.to_swift(BufferLogger(), remove_func=os.unlink)
39 assert not os.path.exists(path), "to_swift failed to move files"
40
41- def file_exists(self, content_id):
42+ def file_exists(self, content_id, suffix=None):
43 container, name = swift.swift_location(content_id)
44+ if suffix:
45+ name += suffix
46 with swift.connection() as swift_connection:
47 try:
48 swift_connection.head_object(container, name)
49@@ -739,6 +745,58 @@
50 with swift.connection() as swift_connection:
51 swift_connection.delete_object(container, name)
52
53+ def test_delete_unwanted_files_handles_segments(self):
54+ # Large files are handled by Swift as multiple segments joined
55+ # by a manifest. GC treats the segments like the original file.
56+ switch_dbuser('testadmin')
57+ content = 'uploading to swift bigly'
58+ big1_lfa = LibraryFileAlias.get(self.client.addFile(
59+ 'foo.txt', len(content), StringIO(content), 'text/plain'))
60+ big1_id = big1_lfa.contentID
61+
62+ big2_lfa = LibraryFileAlias.get(self.client.addFile(
63+ 'bar.txt', len(content), StringIO(content), 'text/plain'))
64+ big2_id = big2_lfa.contentID
65+ transaction.commit()
66+
67+ for lfc_id in (big1_id, big2_id):
68+ # Make the files old so they don't look in-progress.
69+ os.utime(swift.filesystem_path(lfc_id), (0, 0))
70+
71+ switch_dbuser(config.librarian_gc.dbuser)
72+
73+ # Force the files to be segmented as if they were large.
74+ with monkey_patch(swift, MAX_SWIFT_OBJECT_SIZE=4):
75+ swift.to_swift(BufferLogger(), remove_func=os.unlink)
76+
77+ def segment_existence(lfc_id):
78+ return [
79+ self.file_exists(lfc_id, suffix=suffix)
80+ for suffix in (None, '/0000', '/0001')]
81+
82+ # Both files and their segments exist in Swift.
83+ self.assertEqual([True, True, True], segment_existence(big1_id))
84+ self.assertEqual([True, True, True], segment_existence(big2_id))
85+
86+ # All the segments survive the first purge.
87+ with self.librariangc_thinking_it_is_tomorrow():
88+ librariangc.delete_unwanted_files(self.con)
89+ self.assertEqual([True, True, True], segment_existence(big1_id))
90+ self.assertEqual([True, True, True], segment_existence(big2_id))
91+
92+ # Remove the first file from the DB.
93+ content = big1_lfa.content
94+ Store.of(big1_lfa).remove(big1_lfa)
95+ Store.of(content).remove(content)
96+ transaction.commit()
97+
98+ # The first file and its segments are removed, but the second is
99+ # intact.
100+ with self.librariangc_thinking_it_is_tomorrow():
101+ librariangc.delete_unwanted_files(self.con)
102+ self.assertEqual([False, False, False], segment_existence(big1_id))
103+ self.assertEqual([True, True, True], segment_existence(big2_id))
104+
105
106 class TestBlobCollection(TestCase):
107 layer = LaunchpadZopelessLayer