Ubuntu Developer Portal

Merge lp:~dholbach/developer-ubuntu-com/1471160 into lp:developer-ubuntu-com

  1. 1471160
  2. Merge into stable
Proposed by Daniel Holbach on 2015-07-14
Status: Rejected
Rejected by: Daniel Holbach on 2015-07-16
Proposed branch: lp:~dholbach/developer-ubuntu-com/1471160
Merge into: lp:developer-ubuntu-com
Diff against target: 341 lines (+293/-0)
5 files modified
developer_portal/admin.py (+7/-0)
developer_portal/management/commands/import-snappy-branches.py (+214/-0)
developer_portal/migrations/0003_add_snappy_docs_branches.py (+60/-0)
developer_portal/models.py (+10/-0)
requirements.txt (+2/-0)
To merge this branch: bzr merge lp:~dholbach/developer-ubuntu-com/1471160
Reviewer Review Type Date Requested Status
Ubuntu App Developer site developers 2015-07-14 Pending
Review via email: mp+264673@code.launchpad.net
To post a comment you must log in.
Daniel Holbach (dholbach) wrote :

First cut of importing snappy docs into the developer site.

lp:~dholbach/developer-ubuntu-com/1471160 updated on 2015-07-16
133. By Daniel Holbach on 2015-07-14

fix BeautifulSoup initialisation warning

134. By Daniel Holbach on 2015-07-14

generate cms path from imported snappy file, add comment

135. By Daniel Holbach on 2015-07-15

fix admin view of SnappyDocsBranch

136. By Daniel Holbach on 2015-07-16

merged lp:~davidc3/developer-ubuntu-com/1471160_publication

137. By Daniel Holbach on 2015-07-16

add FIXME entries

Unmerged revisions

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'developer_portal/admin.py'
2--- developer_portal/admin.py 2014-12-01 15:11:25 +0000
3+++ developer_portal/admin.py 2015-07-16 13:46:03 +0000
4@@ -3,6 +3,8 @@
5 from reversion.models import Revision, Version
6 from reversion.admin import VersionAdmin
7
8+from .models import SnappyDocsBranch
9+
10 __all__ = (
11 )
12
13@@ -22,3 +24,8 @@
14
15 admin.site.register(Version, VersionAdmin)
16
17+class SnappyDocsBranchAdmin(admin.ModelAdmin):
18+ list_display = ('branch_origin', 'path_alias')
19+ list_filter = ('branch_origin', 'path_alias')
20+
21+admin.site.register(SnappyDocsBranch, SnappyDocsBranchAdmin)
22
23=== added file 'developer_portal/management/commands/import-snappy-branches.py'
24--- developer_portal/management/commands/import-snappy-branches.py 1970-01-01 00:00:00 +0000
25+++ developer_portal/management/commands/import-snappy-branches.py 2015-07-16 13:46:03 +0000
26@@ -0,0 +1,214 @@
27+from django.core.management.base import NoArgsCommand
28+
29+from bs4 import BeautifulSoup
30+import codecs
31+import glob
32+import logging
33+import markdown
34+import os
35+import re
36+import shutil
37+import subprocess
38+import tempfile
39+
40+from developer_portal.models import SnappyDocsBranch
41+
42+RELEASE_PAGES = {}
43+
44+
45+class MarkdownFile():
46+ html = None
47+ cms_path = ''
48+
49+ def __init__(self, fn):
50+ self.fn = fn
51+ self.slug = slugify(self.fn)
52+ with codecs.open(self.fn, 'r', encoding='utf-8') as f:
53+ self.html = markdown.markdown(f.read(), output_format="html5")
54+ self.release_alias = self._get_release_alias()
55+ self.title = self._read_title()
56+ self._remove_body_and_html_tags()
57+ self._use_developer_site_style()
58+
59+ def _get_release_alias(self):
60+ alias = re.findall(r'/tmp/tmp\S+?/(\S+?)/docs/\S+?', self.fn)
61+ return alias[0]
62+
63+ def _read_title(self):
64+ soup = BeautifulSoup(self.html, 'html5lib')
65+ if soup.title:
66+ return soup.title.text
67+ if soup.h1:
68+ return soup.h1.text
69+ return slugify(self.fn).replace('-', ' ').title()
70+
71+ def _remove_body_and_html_tags(self):
72+ self.html = re.sub(r"<html>\n\s<body>\n", "", self.html,
73+ flags=re.MULTILINE)
74+ self.html = re.sub(r"\s<\/body>\n<\/html>", "", self.html,
75+ flags=re.MULTILINE)
76+
77+ def _use_developer_site_style(self):
78+ # Make sure the reader knows which documentation she is browsing
79+ if self.release_alias != "current":
80+ begin = (u"<div class=\"row no-border\">\n"
81+ "<div class=\"box pull-three three-col\">"
82+ "<p>You are browsing the Snappy <code>%s</code> "
83+ "documentation.</p>"
84+ "<p><a href=\"/snappy/guides/current/%s\">"
85+ "Back to the latest stable release &rsaquo;"
86+ "</a></p></div>\n"
87+ "<div class=\"eight-col\">\n") % (self.release_alias,
88+ self.slug, )
89+ else:
90+ begin = (u"<div class=\"row no-border\">"
91+ "\n<div class=\"eight-col\">\n")
92+ end = u"</div>\n</div>"
93+ self.html = begin + self.html + end
94+ self.html = self.html.replace(
95+ "<pre><code>",
96+ "</div><div class=\"twelve-col\"><pre><code>")
97+ self.html = self.html.replace(
98+ "</code></pre>",
99+ "</code></pre></div><div class=\"eight-col\">")
100+
101+ def replace_links(self, titles):
102+ for title in titles:
103+ url = u"/snappy/guides/%s/%s" % (
104+ self.release_alias, slugify(title))
105+ link = u"<a href=\"%s\">%s</a>" % (url, titles[title])
106+ self.html = self.html.replace(os.path.basename(title), link)
107+
108+ def publish(self):
109+ '''Publishes pages in their branch alias namespace.'''
110+ from cms.api import create_page, add_plugin
111+
112+ page_title = self.title
113+
114+ if self.release_alias == "current":
115+ # Add a guides/<page> redirect to guides/current/<page>
116+ page = create_page(
117+ self.title, "default.html", "en",
118+ slug=self.slug, parent=RELEASE_PAGES['guides_page'],
119+ in_navigation=True, position="last-child",
120+ redirect="/snappy/guides/current/%s" % (self.slug))
121+ page.publish('en')
122+ else:
123+ page_title += " (%s)" % (self.release_alias,)
124+
125+ page = create_page(
126+ page_title, "default.html", "en", slug=self.slug,
127+ menu_title=self.title, parent=RELEASE_PAGES[self.release_alias],
128+ in_navigation=True, position="last-child")
129+ placeholder = page.placeholders.get()
130+ add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=self.html)
131+ page.publish('en')
132+
133+
134+def slugify(filename):
135+ return os.path.basename(filename).replace('.md', '')
136+
137+
138+def get_branch_from_lp(origin, alias):
139+ return subprocess.call([
140+ 'bzr', 'checkout', '--lightweight', origin, alias])
141+
142+
143+class LocalBranch():
144+ titles = {}
145+
146+ def __init__(self, dirname):
147+ self.dirname = dirname
148+ self.docs_path = os.path.join(self.dirname, 'docs')
149+ self.doc_fns = glob.glob(self.docs_path+'/*.md')
150+ self.md_files = []
151+
152+ def import_markdown(self):
153+ for doc_fn in self.doc_fns:
154+ md_file = MarkdownFile(doc_fn)
155+ self.md_files += [md_file]
156+ self.titles[md_file.fn] = md_file.title
157+ for md_file in self.md_files:
158+ md_file.replace_links(self.titles)
159+ md_file.publish()
160+
161+
162+# FIXME:
163+# - we retrieve the old article somehow
164+# - then find the Raw HTML plugin and
165+# - replace the html in there
166+# - also: remove pages we don't need anymore
167+# - add new ones
168+# - make sure we can do that for different sets of docs with different pages
169+#
170+def remove_old_pages():
171+ '''Removes all pages in snappy/guides, created by the importer.'''
172+ from cms.models import Title, Page
173+
174+ pages_to_remove = []
175+ for g in Title.objects.select_related('page__id').filter(
176+ path__regex="snappy/guides/.*"):
177+ pages_to_remove.append(g.page.id)
178+ Page.objects.filter(id__in=pages_to_remove, created_by="script").delete()
179+
180+
181+def refresh_landing_page(release_alias):
182+ '''Creates a branch page at snappy/guides/<branch alias>.'''
183+ from cms.api import create_page
184+ from cms.models import Title
185+
186+ guides_page = Title.objects.filter(
187+ path="snappy/guides", published=True,
188+ language="en", publisher_is_draft=True)[0]
189+ RELEASE_PAGES['guides_page'] = guides_page.page
190+
191+ if release_alias == "current":
192+ redirect = "/snappy/guides"
193+ else:
194+ redirect = None
195+ new_release_page = create_page(
196+ release_alias, "default.html", "en", slug=release_alias,
197+ parent=RELEASE_PAGES['guides_page'], in_navigation=False,
198+ position="last-child", redirect=redirect)
199+ # FIXME Page needs content
200+ # placeholder = new_release_page.placeholders.get()
201+ # add_plugin(placeholder, 'RawHtmlPlugin', 'en', body="<html goes here>")
202+ new_release_page.publish('en')
203+ RELEASE_PAGES[release_alias] = new_release_page
204+
205+
206+def import_branches():
207+ if not SnappyDocsBranch.objects.count():
208+ logging.error('No Snappy branches registered in the '
209+ 'SnappyDocsBranch table yet.')
210+ return
211+ # FIXME: Do the removal part last. Else we might end up in situations
212+ # where some code breaks and we stay in a state without articles.
213+ remove_old_pages()
214+ tempdir = tempfile.mkdtemp()
215+ pwd = os.getcwd()
216+ os.chdir(tempdir)
217+ for branch in SnappyDocsBranch.objects.all():
218+ if get_branch_from_lp(branch.branch_origin, branch.path_alias) != 0:
219+ logging.error(
220+ 'Could not check out branch "%s".' % branch.branch_origin)
221+ shutil.rmtree(os.path.join(tempdir, branch.path_alias))
222+ break
223+ refresh_landing_page(branch.path_alias)
224+ os.chdir(pwd)
225+ for local_branch in [a for a in glob.glob(tempdir+'/*')
226+ if os.path.isdir(a)]:
227+ branch = LocalBranch(local_branch)
228+ branch.import_markdown()
229+ shutil.rmtree(tempdir)
230+
231+
232+class Command(NoArgsCommand):
233+ help = "Import Snappy branches for documentation."
234+
235+ def handle_noargs(self, **options):
236+ logging.basicConfig(
237+ level=logging.DEBUG,
238+ format='%(asctime)s %(levelname)-8s %(message)s',
239+ datefmt='%F %T')
240+ import_branches()
241
242=== added file 'developer_portal/migrations/0003_add_snappy_docs_branches.py'
243--- developer_portal/migrations/0003_add_snappy_docs_branches.py 1970-01-01 00:00:00 +0000
244+++ developer_portal/migrations/0003_add_snappy_docs_branches.py 2015-07-16 13:46:03 +0000
245@@ -0,0 +1,60 @@
246+# -*- coding: utf-8 -*-
247+from south.utils import datetime_utils as datetime
248+from south.db import db
249+from south.v2 import SchemaMigration
250+from django.db import models
251+
252+
253+class Migration(SchemaMigration):
254+
255+ def forwards(self, orm):
256+ # Adding model 'SnappyDocsBranch'
257+ db.create_table(u'developer_portal_snappydocsbranch', (
258+ (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
259+ ('branch_origin', self.gf('django.db.models.fields.CharField')(max_length=200)),
260+ ('path_alias', self.gf('django.db.models.fields.CharField')(max_length=20)),
261+ ))
262+ db.send_create_signal(u'developer_portal', ['SnappyDocsBranch'])
263+
264+
265+ def backwards(self, orm):
266+ # Deleting model 'SnappyDocsBranch'
267+ db.delete_table(u'developer_portal_snappydocsbranch')
268+
269+
270+ models = {
271+ 'cms.cmsplugin': {
272+ 'Meta': {'object_name': 'CMSPlugin'},
273+ 'changed_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
274+ 'creation_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
275+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
276+ 'language': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
277+ 'level': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
278+ 'lft': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
279+ 'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.CMSPlugin']", 'null': 'True', 'blank': 'True'}),
280+ 'placeholder': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.Placeholder']", 'null': 'True'}),
281+ 'plugin_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
282+ 'position': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True', 'blank': 'True'}),
283+ 'rght': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
284+ 'tree_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'})
285+ },
286+ 'cms.placeholder': {
287+ 'Meta': {'object_name': 'Placeholder'},
288+ 'default_width': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True'}),
289+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
290+ 'slot': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'})
291+ },
292+ u'developer_portal.rawhtml': {
293+ 'Meta': {'object_name': 'RawHtml'},
294+ 'body': ('django.db.models.fields.TextField', [], {}),
295+ u'cmsplugin_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['cms.CMSPlugin']", 'unique': 'True', 'primary_key': 'True'})
296+ },
297+ u'developer_portal.snappydocsbranch': {
298+ 'Meta': {'object_name': 'SnappyDocsBranch'},
299+ 'branch_origin': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
300+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
301+ 'path_alias': ('django.db.models.fields.CharField', [], {'max_length': '20'})
302+ }
303+ }
304+
305+ complete_apps = ['developer_portal']
306\ No newline at end of file
307
308=== modified file 'developer_portal/models.py'
309--- developer_portal/models.py 2015-01-19 16:29:47 +0000
310+++ developer_portal/models.py 2015-07-16 13:46:03 +0000
311@@ -1,3 +1,5 @@
312+from django.db import models
313+from django.utils.translation import ugettext_lazy as _
314
315 from cms.models import CMSPlugin
316 from djangocms_text_ckeditor.html import extract_images
317@@ -13,3 +15,11 @@
318 body = extract_images(body, self)
319 self.body = body
320 AbstractText.save(self, *args, **kwargs)
321+
322+
323+class SnappyDocsBranch(models.Model):
324+ branch_origin = models.CharField(max_length=200,
325+ help_text=_('Launchpad branch location, ie: lp:snappy/15.04'))
326+ path_alias = models.CharField(max_length=20,
327+ help_text=_('Path alias we want to use for the docs, '
328+ 'ie "15.04" or "latest", etc.'))
329
330=== modified file 'requirements.txt'
331--- requirements.txt 2015-02-26 15:53:30 +0000
332+++ requirements.txt 2015-07-16 13:46:03 +0000
333@@ -1,6 +1,8 @@
334 Django==1.6.8
335+Markdown==2.6.2
336 South==1.0.1
337 Pillow==2.6.1
338+beautifulsoup4==4.4.0
339 cmsplugin-zinnia==0.6
340 dj-database-url==0.3.0
341 django-admin-enhancer==0.1.3.1

Subscribers

People subscribed via source and target branches