Turku Backup System

Merge lp:~cjwatson/turku/turku-api-preserve-hard-links into lp:turku/turku-api

  1. turku-api-preserve-hard-links
  2. Merge into turku-api
Proposed by Colin Watson
Status: Merged
Approved by: Nick Moffitt
Approved revision: 59
Merged at revision: 59
Proposed branch: lp:~cjwatson/turku/turku-api-preserve-hard-links
Merge into: lp:turku/turku-api
Diff against target: 174 lines (+126/-0)
3 files modified
turku_api/migrations/0024_auto__add_field_source_preserve_hard_links.py (+118/-0)
turku_api/models.py (+4/-0)
turku_api/views.py (+4/-0)
To merge this branch: bzr merge lp:~cjwatson/turku/turku-api-preserve-hard-links
Reviewer Review Type Date Requested Status
Turku Pending
Review via email: mp+374827@code.launchpad.net

Commit message

Add a per-source option to preserve hard links.

Description of the change

This can be expensive, so we shouldn't do it for every source. However, the git service uses hard links when creating subordinate repositories, and not preserving these may cause a significant amount of backup bloat.

To post a comment you must log in.
Revision history for this message
🤖 Canonical IS Merge Bot (canonical-is-mergebot) wrote :

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

Revision history for this message
Colin Watson (cjwatson) wrote :

This is for https://portal.admin.canonical.com/C122244.

Revision history for this message
🤖 Canonical IS Merge Bot (canonical-is-mergebot) wrote :

Change successfully merged at revision 59

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'turku_api/migrations/0024_auto__add_field_source_preserve_hard_links.py'
2--- turku_api/migrations/0024_auto__add_field_source_preserve_hard_links.py 1970-01-01 00:00:00 +0000
3+++ turku_api/migrations/0024_auto__add_field_source_preserve_hard_links.py 2019-10-29 10:09:07 +0000
4@@ -0,0 +1,118 @@
5+# -*- coding: utf-8 -*-
6+from south.utils import datetime_utils as datetime
7+from south.db import db
8+from south.v2 import SchemaMigration
9+from django.db import models
10+
11+
12+class Migration(SchemaMigration):
13+
14+ def forwards(self, orm):
15+ # Adding field 'Source.preserve_hard_links'
16+ db.add_column(u'turku_api_source', 'preserve_hard_links',
17+ self.gf('django.db.models.fields.BooleanField')(default=False),
18+ keep_default=False)
19+
20+
21+ def backwards(self, orm):
22+ # Deleting field 'Source.preserve_hard_links'
23+ db.delete_column(u'turku_api_source', 'preserve_hard_links')
24+
25+
26+ models = {
27+ u'turku_api.auth': {
28+ 'Meta': {'object_name': 'Auth'},
29+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
30+ 'comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
31+ 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
32+ 'id': ('turku_api.models.UuidPrimaryKeyField', [], {'default': "'fd39b2a6-60bb-4ad4-b5b3-7353a42f777e'", 'max_length': '36', 'primary_key': 'True'}),
33+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}),
34+ 'secret_hash': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
35+ 'secret_type': ('django.db.models.fields.CharField', [], {'max_length': '200'})
36+ },
37+ u'turku_api.backuplog': {
38+ 'Meta': {'object_name': 'BackupLog'},
39+ 'date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
40+ 'date_begin': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
41+ 'date_end': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
42+ 'id': ('turku_api.models.UuidPrimaryKeyField', [], {'default': "'6c1c2365-0bcb-4ec3-9088-f606b79a75ee'", 'max_length': '36', 'primary_key': 'True'}),
43+ 'snapshot': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
44+ 'source': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['turku_api.Source']"}),
45+ 'storage': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['turku_api.Storage']", 'null': 'True', 'blank': 'True'}),
46+ 'success': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
47+ 'summary': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
48+ },
49+ u'turku_api.filterset': {
50+ 'Meta': {'object_name': 'FilterSet'},
51+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
52+ 'comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
53+ 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
54+ 'filters': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
55+ 'id': ('turku_api.models.UuidPrimaryKeyField', [], {'default': "'a60897e7-e1b9-4a30-8a23-419245be2ede'", 'max_length': '36', 'primary_key': 'True'}),
56+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'})
57+ },
58+ u'turku_api.machine': {
59+ 'Meta': {'object_name': 'Machine'},
60+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
61+ 'auth': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['turku_api.Auth']"}),
62+ 'comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
63+ 'date_checked_in': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
64+ 'date_registered': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
65+ 'date_updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
66+ 'environment_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
67+ 'id': ('turku_api.models.UuidPrimaryKeyField', [], {'default': "'3e6bc992-fec5-4e51-823a-0a70dc9607a0'", 'max_length': '36', 'primary_key': 'True'}),
68+ 'published': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
69+ 'secret_hash': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
70+ 'service_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
71+ 'ssh_public_key': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
72+ 'storage': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['turku_api.Storage']"}),
73+ 'unit_name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
74+ 'uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'})
75+ },
76+ u'turku_api.source': {
77+ 'Meta': {'unique_together': "(('machine', 'name'),)", 'object_name': 'Source'},
78+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
79+ 'bwlimit': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
80+ 'comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
81+ 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
82+ 'date_last_backed_up': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
83+ 'date_next_backup': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
84+ 'date_updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
85+ 'exclude': ('django.db.models.fields.CharField', [], {'default': "'[]'", 'max_length': '2048'}),
86+ 'filter': ('django.db.models.fields.CharField', [], {'default': "'[]'", 'max_length': '2048'}),
87+ 'frequency': ('django.db.models.fields.CharField', [], {'default': "'daily'", 'max_length': '200'}),
88+ 'id': ('turku_api.models.UuidPrimaryKeyField', [], {'default': "'76095f45-08a2-4b63-9ed7-0e8d3a3aa820'", 'max_length': '36', 'primary_key': 'True'}),
89+ 'large_modifying_files': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
90+ 'large_rotating_files': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
91+ 'machine': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['turku_api.Machine']"}),
92+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
93+ 'path': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
94+ 'preserve_hard_links': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
95+ 'published': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
96+ 'retention': ('django.db.models.fields.CharField', [], {'default': "'last 5 days, earliest of month'", 'max_length': '200'}),
97+ 'shared_service': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
98+ 'snapshot_mode': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
99+ 'success': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
100+ },
101+ u'turku_api.storage': {
102+ 'Meta': {'object_name': 'Storage'},
103+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
104+ 'auth': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['turku_api.Auth']"}),
105+ 'comment': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
106+ 'date_checked_in': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
107+ 'date_registered': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
108+ 'date_updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
109+ 'id': ('turku_api.models.UuidPrimaryKeyField', [], {'default': "'5eace5c5-d46e-4d43-adab-efae9d36f3ac'", 'max_length': '36', 'primary_key': 'True'}),
110+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}),
111+ 'published': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
112+ 'secret_hash': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
113+ 'space_available': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
114+ 'space_total': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
115+ 'ssh_ping_host': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
116+ 'ssh_ping_host_keys': ('django.db.models.fields.CharField', [], {'default': "'[]'", 'max_length': '65536'}),
117+ 'ssh_ping_port': ('django.db.models.fields.PositiveIntegerField', [], {}),
118+ 'ssh_ping_user': ('django.db.models.fields.CharField', [], {'max_length': '200'})
119+ }
120+ }
121+
122+ complete_apps = ['turku_api']
123\ No newline at end of file
124
125=== modified file 'turku_api/models.py'
126--- turku_api/models.py 2017-06-06 15:13:28 +0000
127+++ turku_api/models.py 2019-10-29 10:09:07 +0000
128@@ -342,6 +342,10 @@
129 max_length=200, choices=SNAPSHOT_MODES,
130 help_text='Override the storage unit\'s snapshot logic and use an explicit snapshot mode for this source.',
131 )
132+ preserve_hard_links = models.BooleanField(
133+ default=False,
134+ help_text='Whether to preserve hard links when backing up this source.',
135+ )
136 shared_service = models.BooleanField(
137 default=False,
138 help_text='Whether this source is part of a shared service of multiple machines to be backed up.',
139
140=== modified file 'turku_api/views.py'
141--- turku_api/views.py 2015-07-30 22:41:42 +0000
142+++ turku_api/views.py 2019-10-29 10:09:07 +0000
143@@ -294,6 +294,7 @@
144 'path', 'frequency', 'retention',
145 'comment', 'shared_service', 'large_rotating_files',
146 'large_modifying_files', 'bwlimit', 'snapshot_mode',
147+ 'preserve_hard_links',
148 ):
149 if (k in req_sources[s.name]) and (getattr(s, k) != req_sources[s.name][k]):
150 setattr(s, k, req_sources[s.name][k])
151@@ -328,6 +329,7 @@
152 'path', 'frequency', 'retention',
153 'comment', 'shared_service', 'large_rotating_files',
154 'large_modifying_files', 'bwlimit', 'snapshot_mode',
155+ 'preserve_hard_links',
156 ):
157 if k not in req_sources[s.name]:
158 continue
159@@ -399,6 +401,7 @@
160 'large_rotating_files': s.large_rotating_files,
161 'large_modifying_files': s.large_modifying_files,
162 'snapshot_mode': s.snapshot_mode,
163+ 'preserve_hard_links': s.preserve_hard_links,
164 'storage': {
165 'name': s.machine.storage.name,
166 'ssh_ping_host': s.machine.storage.ssh_ping_host,
167@@ -473,6 +476,7 @@
168 'large_rotating_files': s.large_rotating_files,
169 'large_modifying_files': s.large_modifying_files,
170 'snapshot_mode': s.snapshot_mode,
171+ 'preserve_hard_links': s.preserve_hard_links,
172 'storage': {
173 'name': s.machine.storage.name,
174 'ssh_ping_host': s.machine.storage.ssh_ping_host,

Subscribers

People subscribed via source and target branches

to all changes: