Merge lp:~cjwatson/launchpad/bmp-job-pruner into lp:launchpad

Proposed by Colin Watson on 2019-01-30
Status: Needs review
Proposed branch: lp:~cjwatson/launchpad/bmp-job-pruner
Merge into: lp:launchpad
Diff against target: 111 lines (+66/-2)
2 files modified
lib/lp/scripts/garbo.py (+20/-1)
lib/lp/scripts/tests/test_garbo.py (+46/-1)
To merge this branch: bzr merge lp:~cjwatson/launchpad/bmp-job-pruner
Reviewer Review Type Date Requested Status
Launchpad code reviewers 2019-01-30 Pending
Review via email: mp+362460@code.launchpad.net

Commit message

Prune old completed BranchMergeProposalJobs.

Description of the change

This accounts for about 3 million out of the 17 million Job rows on production, so it seems worth the effort of pruning.

To post a comment you must log in.

Unmerged revisions

18866. By Colin Watson on 2019-01-30

Prune old completed BranchMergeProposalJobs.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/scripts/garbo.py'
2--- lib/lp/scripts/garbo.py 2018-12-17 14:49:53 +0000
3+++ lib/lp/scripts/garbo.py 2019-01-30 14:10:40 +0000
4@@ -1,4 +1,4 @@
5-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
6+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
7 # GNU Affero General Public License version 3 (see the file LICENSE).
8
9 """Database garbage collection."""
10@@ -1238,6 +1238,24 @@
11 """
12
13
14+class BranchMergeProposalJobPruner(BulkPruner):
15+ """Prune `BranchMergeProposalJob`s that are in a final state and more
16+ than a month old.
17+
18+ When a BranchMergeProposalJob is completed, it gets set to a final
19+ state. These jobs should be pruned from the database after a month.
20+ """
21+ target_table_class = Job
22+ ids_to_prune_query = """
23+ SELECT DISTINCT Job.id
24+ FROM Job, BranchMergeProposalJob
25+ WHERE
26+ Job.id = BranchMergeProposalJob.job
27+ AND Job.date_finished < CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
28+ - CAST('30 days' AS interval)
29+ """
30+
31+
32 class SnapBuildJobPruner(BulkPruner):
33 """Prune `SnapBuildJob`s that are in a final state and more than a month
34 old.
35@@ -1927,6 +1945,7 @@
36 tunable_loops = [
37 AnswerContactPruner,
38 BranchJobPruner,
39+ BranchMergeProposalJobPruner,
40 BugNotificationPruner,
41 BugWatchActivityPruner,
42 CodeImportEventPruner,
43
44=== modified file 'lib/lp/scripts/tests/test_garbo.py'
45--- lib/lp/scripts/tests/test_garbo.py 2018-12-17 14:49:53 +0000
46+++ lib/lp/scripts/tests/test_garbo.py 2019-01-30 14:10:40 +0000
47@@ -1,4 +1,4 @@
48-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
49+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
50 # GNU Affero General Public License version 3 (see the file LICENSE).
51
52 """Test the database garbage collector."""
53@@ -61,6 +61,7 @@
54 BranchJob,
55 BranchUpgradeJob,
56 )
57+from lp.code.model.branchmergeproposaljob import BranchMergeProposalJob
58 from lp.code.model.codeimportevent import CodeImportEvent
59 from lp.code.model.codeimportresult import CodeImportResult
60 from lp.code.model.diff import Diff
61@@ -957,6 +958,50 @@
62 switch_dbuser('testadmin')
63 self.assertEqual(store.find(BranchJob).count(), 1)
64
65+ def test_BranchMergeProposalJobPruner(self):
66+ # Garbo should remove jobs completed over 30 days ago.
67+ switch_dbuser('testadmin')
68+ store = IMasterStore(Job)
69+
70+ bmp = self.factory.makeBranchMergeProposal()
71+ bmp_job = removeSecurityProxy(bmp.next_preview_diff_job)
72+ bmp_job.job.date_finished = THIRTY_DAYS_AGO
73+
74+ self.assertEqual(
75+ store.find(
76+ BranchMergeProposalJob,
77+ BranchMergeProposalJob.branch_merge_proposal ==
78+ bmp.id).count(),
79+ 1)
80+
81+ self.runDaily()
82+
83+ switch_dbuser('testadmin')
84+ self.assertEqual(
85+ store.find(
86+ BranchMergeProposalJob,
87+ BranchMergeProposalJob.branch_merge_proposal ==
88+ bmp.id).count(),
89+ 0)
90+
91+ def test_BranchMergeProposalJobPruner_doesnt_prune_recent_jobs(self):
92+ # Check to make sure the garbo doesn't remove jobs that aren't more
93+ # than thirty days old.
94+ switch_dbuser('testadmin')
95+ store = IMasterStore(Job)
96+
97+ bmp = self.factory.makeBranchMergeProposal()
98+ bmp_job = removeSecurityProxy(bmp.next_preview_diff_job)
99+ bmp_job.job.date_finished = THIRTY_DAYS_AGO
100+
101+ bmp2 = self.factory.makeBranchMergeProposal()
102+ self.assertIsNotNone(bmp2.next_preview_diff_job)
103+
104+ self.runDaily()
105+
106+ switch_dbuser('testadmin')
107+ self.assertEqual(store.find(BranchMergeProposalJob).count(), 1)
108+
109 def test_GitJobPruner(self):
110 # Garbo should remove jobs completed over 30 days ago.
111 switch_dbuser('testadmin')