Ubuntu Developer Portal

Merge lp:~dholbach/developer-ubuntu-com/1471160 into lp:developer-ubuntu-com

  1. 1471160
  2. Merge into stable
Proposed by Daniel Holbach
Status: Rejected
Rejected by: Daniel Holbach
Proposed branch: lp:~dholbach/developer-ubuntu-com/1471160
Merge into: lp:developer-ubuntu-com
Diff against target: 341 lines (+293/-0)
5 files modified
developer_portal/admin.py (+7/-0)
developer_portal/management/commands/import-snappy-branches.py (+214/-0)
developer_portal/migrations/0003_add_snappy_docs_branches.py (+60/-0)
developer_portal/models.py (+10/-0)
requirements.txt (+2/-0)
To merge this branch: bzr merge lp:~dholbach/developer-ubuntu-com/1471160
Reviewer Review Type Date Requested Status
Ubuntu App Developer site developers Pending
Review via email: mp+264673@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Daniel Holbach (dholbach) wrote :

First cut of importing snappy docs into the developer site.

lp:~dholbach/developer-ubuntu-com/1471160 updated
133. By Daniel Holbach

fix BeautifulSoup initialisation warning

134. By Daniel Holbach

generate cms path from imported snappy file, add comment

135. By Daniel Holbach

fix admin view of SnappyDocsBranch

136. By Daniel Holbach

merged lp:~davidc3/developer-ubuntu-com/1471160_publication

137. By Daniel Holbach

add FIXME entries

Unmerged revisions

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'developer_portal/admin.py'
2--- developer_portal/admin.py 2014-12-01 15:11:25 +0000
3+++ developer_portal/admin.py 2015-07-16 13:46:03 +0000
4@@ -3,6 +3,8 @@
5 from reversion.models import Revision, Version
6 from reversion.admin import VersionAdmin
7
8+from .models import SnappyDocsBranch
9+
10 __all__ = (
11 )
12
13@@ -22,3 +24,8 @@
14
15 admin.site.register(Version, VersionAdmin)
16
17+class SnappyDocsBranchAdmin(admin.ModelAdmin):
18+ list_display = ('branch_origin', 'path_alias')
19+ list_filter = ('branch_origin', 'path_alias')
20+
21+admin.site.register(SnappyDocsBranch, SnappyDocsBranchAdmin)
22
23=== added file 'developer_portal/management/commands/import-snappy-branches.py'
24--- developer_portal/management/commands/import-snappy-branches.py 1970-01-01 00:00:00 +0000
25+++ developer_portal/management/commands/import-snappy-branches.py 2015-07-16 13:46:03 +0000
26@@ -0,0 +1,214 @@
27+from django.core.management.base import NoArgsCommand
28+
29+from bs4 import BeautifulSoup
30+import codecs
31+import glob
32+import logging
33+import markdown
34+import os
35+import re
36+import shutil
37+import subprocess
38+import tempfile
39+
40+from developer_portal.models import SnappyDocsBranch
41+
42+RELEASE_PAGES = {}
43+
44+
45+class MarkdownFile():
46+ html = None
47+ cms_path = ''
48+
49+ def __init__(self, fn):
50+ self.fn = fn
51+ self.slug = slugify(self.fn)
52+ with codecs.open(self.fn, 'r', encoding='utf-8') as f:
53+ self.html = markdown.markdown(f.read(), output_format="html5")
54+ self.release_alias = self._get_release_alias()
55+ self.title = self._read_title()
56+ self._remove_body_and_html_tags()
57+ self._use_developer_site_style()
58+
59+ def _get_release_alias(self):
60+ alias = re.findall(r'/tmp/tmp\S+?/(\S+?)/docs/\S+?', self.fn)
61+ return alias[0]
62+
63+ def _read_title(self):
64+ soup = BeautifulSoup(self.html, 'html5lib')
65+ if soup.title:
66+ return soup.title.text
67+ if soup.h1:
68+ return soup.h1.text
69+ return slugify(self.fn).replace('-', ' ').title()
70+
71+ def _remove_body_and_html_tags(self):
72+ self.html = re.sub(r"<html>\n\s<body>\n", "", self.html,
73+ flags=re.MULTILINE)
74+ self.html = re.sub(r"\s<\/body>\n<\/html>", "", self.html,
75+ flags=re.MULTILINE)
76+
77+ def _use_developer_site_style(self):
78+ # Make sure the reader knows which documentation she is browsing
79+ if self.release_alias != "current":
80+ begin = (u"<div class=\"row no-border\">\n"
81+ "<div class=\"box pull-three three-col\">"
82+ "<p>You are browsing the Snappy <code>%s</code> "
83+ "documentation.</p>"
84+ "<p><a href=\"/snappy/guides/current/%s\">"
85+ "Back to the latest stable release &rsaquo;"
86+ "</a></p></div>\n"
87+ "<div class=\"eight-col\">\n") % (self.release_alias,
88+ self.slug, )
89+ else:
90+ begin = (u"<div class=\"row no-border\">"
91+ "\n<div class=\"eight-col\">\n")
92+ end = u"</div>\n</div>"
93+ self.html = begin + self.html + end
94+ self.html = self.html.replace(
95+ "<pre><code>",
96+ "</div><div class=\"twelve-col\"><pre><code>")
97+ self.html = self.html.replace(
98+ "</code></pre>",
99+ "</code></pre></div><div class=\"eight-col\">")
100+
101+ def replace_links(self, titles):
102+ for title in titles:
103+ url = u"/snappy/guides/%s/%s" % (
104+ self.release_alias, slugify(title))
105+ link = u"<a href=\"%s\">%s</a>" % (url, titles[title])
106+ self.html = self.html.replace(os.path.basename(title), link)
107+
108+ def publish(self):
109+ '''Publishes pages in their branch alias namespace.'''
110+ from cms.api import create_page, add_plugin
111+
112+ page_title = self.title
113+
114+ if self.release_alias == "current":
115+ # Add a guides/<page> redirect to guides/current/<page>
116+ page = create_page(
117+ self.title, "default.html", "en",
118+ slug=self.slug, parent=RELEASE_PAGES['guides_page'],
119+ in_navigation=True, position="last-child",
120+ redirect="/snappy/guides/current/%s" % (self.slug))
121+ page.publish('en')
122+ else:
123+ page_title += " (%s)" % (self.release_alias,)
124+
125+ page = create_page(
126+ page_title, "default.html", "en", slug=self.slug,
127+ menu_title=self.title, parent=RELEASE_PAGES[self.release_alias],
128+ in_navigation=True, position="last-child")
129+ placeholder = page.placeholders.get()
130+ add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=self.html)
131+ page.publish('en')
132+
133+
134+def slugify(filename):
135+ return os.path.basename(filename).replace('.md', '')
136+
137+
138+def get_branch_from_lp(origin, alias):
139+ return subprocess.call([
140+ 'bzr', 'checkout', '--lightweight', origin, alias])
141+
142+
143+class LocalBranch():
144+ titles = {}
145+
146+ def __init__(self, dirname):
147+ self.dirname = dirname
148+ self.docs_path = os.path.join(self.dirname, 'docs')
149+ self.doc_fns = glob.glob(self.docs_path+'/*.md')
150+ self.md_files = []
151+
152+ def import_markdown(self):
153+ for doc_fn in self.doc_fns:
154+ md_file = MarkdownFile(doc_fn)
155+ self.md_files += [md_file]
156+ self.titles[md_file.fn] = md_file.title
157+ for md_file in self.md_files:
158+ md_file.replace_links(self.titles)
159+ md_file.publish()
160+
161+
162+# FIXME:
163+# - we retrieve the old article somehow
164+# - then find the Raw HTML plugin and
165+# - replace the html in there
166+# - also: remove pages we don't need anymore
167+# - add new ones
168+# - make sure we can do that for different sets of docs with different pages
169+#
170+def remove_old_pages():
171+ '''Removes all pages in snappy/guides, created by the importer.'''
172+ from cms.models import Title, Page
173+
174+ pages_to_remove = []
175+ for g in Title.objects.select_related('page__id').filter(
176+ path__regex="snappy/guides/.*"):
177+ pages_to_remove.append(g.page.id)
178+ Page.objects.filter(id__in=pages_to_remove, created_by="script").delete()
179+
180+
181+def refresh_landing_page(release_alias):
182+ '''Creates a branch page at snappy/guides/<branch alias>.'''
183+ from cms.api import create_page
184+ from cms.models import Title
185+
186+ guides_page = Title.objects.filter(
187+ path="snappy/guides", published=True,
188+ language="en", publisher_is_draft=True)[0]
189+ RELEASE_PAGES['guides_page'] = guides_page.page
190+
191+ if release_alias == "current":
192+ redirect = "/snappy/guides"
193+ else:
194+ redirect = None
195+ new_release_page = create_page(
196+ release_alias, "default.html", "en", slug=release_alias,
197+ parent=RELEASE_PAGES['guides_page'], in_navigation=False,
198+ position="last-child", redirect=redirect)
199+ # FIXME Page needs content
200+ # placeholder = new_release_page.placeholders.get()
201+ # add_plugin(placeholder, 'RawHtmlPlugin', 'en', body="<html goes here>")
202+ new_release_page.publish('en')
203+ RELEASE_PAGES[release_alias] = new_release_page
204+
205+
206+def import_branches():
207+ if not SnappyDocsBranch.objects.count():
208+ logging.error('No Snappy branches registered in the '
209+ 'SnappyDocsBranch table yet.')
210+ return
211+ # FIXME: Do the removal part last. Else we might end up in situations
212+ # where some code breaks and we stay in a state without articles.
213+ remove_old_pages()
214+ tempdir = tempfile.mkdtemp()
215+ pwd = os.getcwd()
216+ os.chdir(tempdir)
217+ for branch in SnappyDocsBranch.objects.all():
218+ if get_branch_from_lp(branch.branch_origin, branch.path_alias) != 0:
219+ logging.error(
220+ 'Could not check out branch "%s".' % branch.branch_origin)
221+ shutil.rmtree(os.path.join(tempdir, branch.path_alias))
222+ break
223+ refresh_landing_page(branch.path_alias)
224+ os.chdir(pwd)
225+ for local_branch in [a for a in glob.glob(tempdir+'/*')
226+ if os.path.isdir(a)]:
227+ branch = LocalBranch(local_branch)
228+ branch.import_markdown()
229+ shutil.rmtree(tempdir)
230+
231+
232+class Command(NoArgsCommand):
233+ help = "Import Snappy branches for documentation."
234+
235+ def handle_noargs(self, **options):
236+ logging.basicConfig(
237+ level=logging.DEBUG,
238+ format='%(asctime)s %(levelname)-8s %(message)s',
239+ datefmt='%F %T')
240+ import_branches()
241
242=== added file 'developer_portal/migrations/0003_add_snappy_docs_branches.py'
243--- developer_portal/migrations/0003_add_snappy_docs_branches.py 1970-01-01 00:00:00 +0000
244+++ developer_portal/migrations/0003_add_snappy_docs_branches.py 2015-07-16 13:46:03 +0000
245@@ -0,0 +1,60 @@
246+# -*- coding: utf-8 -*-
247+from south.utils import datetime_utils as datetime
248+from south.db import db
249+from south.v2 import SchemaMigration
250+from django.db import models
251+
252+
253+class Migration(SchemaMigration):
254+
255+ def forwards(self, orm):
256+ # Adding model 'SnappyDocsBranch'
257+ db.create_table(u'developer_portal_snappydocsbranch', (
258+ (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
259+ ('branch_origin', self.gf('django.db.models.fields.CharField')(max_length=200)),
260+ ('path_alias', self.gf('django.db.models.fields.CharField')(max_length=20)),
261+ ))
262+ db.send_create_signal(u'developer_portal', ['SnappyDocsBranch'])
263+
264+
265+ def backwards(self, orm):
266+ # Deleting model 'SnappyDocsBranch'
267+ db.delete_table(u'developer_portal_snappydocsbranch')
268+
269+
270+ models = {
271+ 'cms.cmsplugin': {
272+ 'Meta': {'object_name': 'CMSPlugin'},
273+ 'changed_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
274+ 'creation_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
275+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
276+ 'language': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
277+ 'level': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
278+ 'lft': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
279+ 'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.CMSPlugin']", 'null': 'True', 'blank': 'True'}),
280+ 'placeholder': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.Placeholder']", 'null': 'True'}),
281+ 'plugin_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
282+ 'position': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True', 'blank': 'True'}),
283+ 'rght': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
284+ 'tree_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'})
285+ },
286+ 'cms.placeholder': {
287+ 'Meta': {'object_name': 'Placeholder'},
288+ 'default_width': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True'}),
289+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
290+ 'slot': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'})
291+ },
292+ u'developer_portal.rawhtml': {
293+ 'Meta': {'object_name': 'RawHtml'},
294+ 'body': ('django.db.models.fields.TextField', [], {}),
295+ u'cmsplugin_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['cms.CMSPlugin']", 'unique': 'True', 'primary_key': 'True'})
296+ },
297+ u'developer_portal.snappydocsbranch': {
298+ 'Meta': {'object_name': 'SnappyDocsBranch'},
299+ 'branch_origin': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
300+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
301+ 'path_alias': ('django.db.models.fields.CharField', [], {'max_length': '20'})
302+ }
303+ }
304+
305+ complete_apps = ['developer_portal']
306\ No newline at end of file
307
308=== modified file 'developer_portal/models.py'
309--- developer_portal/models.py 2015-01-19 16:29:47 +0000
310+++ developer_portal/models.py 2015-07-16 13:46:03 +0000
311@@ -1,3 +1,5 @@
312+from django.db import models
313+from django.utils.translation import ugettext_lazy as _
314
315 from cms.models import CMSPlugin
316 from djangocms_text_ckeditor.html import extract_images
317@@ -13,3 +15,11 @@
318 body = extract_images(body, self)
319 self.body = body
320 AbstractText.save(self, *args, **kwargs)
321+
322+
323+class SnappyDocsBranch(models.Model):
324+ branch_origin = models.CharField(max_length=200,
325+ help_text=_('Launchpad branch location, ie: lp:snappy/15.04'))
326+ path_alias = models.CharField(max_length=20,
327+ help_text=_('Path alias we want to use for the docs, '
328+ 'ie "15.04" or "latest", etc.'))
329
330=== modified file 'requirements.txt'
331--- requirements.txt 2015-02-26 15:53:30 +0000
332+++ requirements.txt 2015-07-16 13:46:03 +0000
333@@ -1,6 +1,8 @@
334 Django==1.6.8
335+Markdown==2.6.2
336 South==1.0.1
337 Pillow==2.6.1
338+beautifulsoup4==4.4.0
339 cmsplugin-zinnia==0.6
340 dj-database-url==0.3.0
341 django-admin-enhancer==0.1.3.1

Subscribers

People subscribed via source and target branches