Merge lp:~developer-ubuntu-com-dev/developer-ubuntu-com/snappy-docs-import into lp:developer-ubuntu-com

[Daniel Holbach]

Add functionality to import external markdown docs (like snappy and snapcraft) automatically into developer site.

[Daniel Holbach (dholbach)]

This is still WIP.

[Daniel Holbach (dholbach)]

David Callé noticed that SnappyMarkdownFile._make_snappy_mods doesn't work as expected.

review: Needs Fixing

[Daniel Holbach (dholbach)]

<davidcalle> dholbach, well, I'm not sure about the "Snappy" name for landing pages. I haven't found anything else. Oh, maybe the script itself should be renamed to not have snappy in its name.

[David Callé (davidc3)]

Added a few changes.

Lots of testing done with a dummy branch on the whole publication, page removal, doc name changes, etc. processes. Everything works great.

[Daniel Holbach (dholbach)]

<dholbach> davidcalle, I read the code again and I think I'm fine with it
 parts of it could be a bit simpler I feel, but for now I think it's good enough

review: Approve

[Daniel Holbach (dholbach)]

<davidcalle> dholbach, same for me. Let's merge it :) I'll try the staging deploy this afternoon.

review: Approve

=== modified file 'developer_portal/admin.py'
--- developer_portal/admin.py	2014-12-01 15:11:25 +0000
+++ developer_portal/admin.py	2015-08-11 12:06:30 +0000
@@ -3,10 +3,20 @@
 from reversion.models import Revision, Version
 from reversion.admin import VersionAdmin
 
+from .models import ExternalDocsBranch
+from django.core.management import call_command
+
 __all__ = (
 )
 
 
+def import_selected_external_docs_branches(modeladmin, request, queryset):
+    for branch in queryset:
+        call_command('import-external-docs-branches', branch.docs_namespace)
+    import_selected_external_docs_branches.short_description = \
+        "Import selected branches"
+
+
 class RevisionAdmin(admin.ModelAdmin):
     list_display = ('date_created', 'user', 'comment')
     list_display_links = ('date_created', )
@@ -15,6 +25,7 @@
 
 admin.site.register(Revision, RevisionAdmin)
 
+
 class VersionAdmin(admin.ModelAdmin):
     list_display = ('content_type', 'object_id')
     list_display_links = ('object_id', )
@@ -22,3 +33,10 @@
 
 admin.site.register(Version, VersionAdmin)
 
+
+class ExternalDocsBranchAdmin(admin.ModelAdmin):
+    list_display = ('lp_origin', 'docs_namespace')
+    list_filter = ('lp_origin', 'docs_namespace')
+    actions = [import_selected_external_docs_branches]
+
+admin.site.register(ExternalDocsBranch, ExternalDocsBranchAdmin)
 
=== added file 'developer_portal/management/commands/import-external-docs-branches.py'
--- developer_portal/management/commands/import-external-docs-branches.py	1970-01-01 00:00:00 +0000
+++ developer_portal/management/commands/import-external-docs-branches.py	2015-08-11 12:06:30 +0000
@@ -0,0 +1,309 @@
+from django.core.management.base import BaseCommand
+
+from cms.api import create_page, add_plugin
+from cms.models import Page, Title
+from cms.utils import page_resolver
+
+from bs4 import BeautifulSoup
+import codecs
+import glob
+import logging
+import markdown
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+from developer_portal.models import ExternalDocsBranch
+
+DOCS_DIRNAME = 'docs'
+
+
+class MarkdownFile:
+    html = None
+
+    def __init__(self, fn, docs_namespace, slug_override=None):
+        self.fn = fn
+        self.docs_namespace = docs_namespace
+        if slug_override:
+            self.slug = slug_override
+        else:
+            self.slug = slugify(self.fn)
+        self.full_url = os.path.join(self.docs_namespace, self.slug)
+        with codecs.open(self.fn, 'r', encoding='utf-8') as f:
+            self.html = markdown.markdown(
+                f.read(),
+                output_format="html5",
+                extensions=['markdown.extensions.tables'])
+        self.release_alias = self._get_release_alias()
+        self.title = self._read_title()
+        self._remove_body_and_html_tags()
+        self._use_developer_site_style()
+
+    def _get_release_alias(self):
+        alias = re.findall(r'/tmp/tmp\S+?/(\S+?)/%s/\S+?' % DOCS_DIRNAME,
+                           self.fn)
+        return alias[0]
+
+    def _read_title(self):
+        soup = BeautifulSoup(self.html, 'html5lib')
+        if soup.title:
+            return soup.title.text
+        if soup.h1:
+            return soup.h1.text
+        return slugify(self.fn).replace('-', ' ').title()
+
+    def _remove_body_and_html_tags(self):
+        self.html = re.sub(r"<html>\n\s<body>\n", "", self.html,
+                           flags=re.MULTILINE)
+        self.html = re.sub(r"\s<\/body>\n<\/html>", "", self.html,
+                           flags=re.MULTILINE)
+
+    def _use_developer_site_style(self):
+        begin = (u"<div class=\"row no-border\">"
+                 "\n<div class=\"eight-col\">\n")
+        end = u"</div>\n</div>"
+        self.html = begin + self.html + end
+        self.html = self.html.replace(
+            "<pre><code>",
+            "</div><div class=\"twelve-col\"><pre><code>")
+        self.html = self.html.replace(
+            "</code></pre>",
+            "</code></pre></div><div class=\"eight-col\">")
+
+    def replace_links(self, titles):
+        for title in titles:
+            url = u"/snappy/guides/%s/%s" % (
+                self.release_alias, slugify(title))
+            link = u"<a href=\"%s\">%s</a>" % (url, titles[title])
+            self.html = self.html.replace(os.path.basename(title), link)
+
+    def publish(self):
+        '''Publishes pages in their branch alias namespace.'''
+        page = get_or_create_page(
+            self.title, full_url=self.full_url, menu_title=self.title,
+            html=self.html)
+        page.publish('en')
+
+
+class SnappyMarkdownFile(MarkdownFile):
+    def __init__(self, fn, docs_namespace):
+        MarkdownFile.__init__(self, fn, docs_namespace)
+        self._make_snappy_mods()
+
+    def _make_snappy_mods(self):
+        # Make sure the reader knows which documentation she is browsing
+        if self.release_alias != 'current':
+            before = (u"<div class=\"row no-border\">\n"
+                      "<div class=\"eight-col\">\n")
+            after = (u"<div class=\"row no-border\">\n"
+                     "<div class=\"box pull-three three-col\">"
+                     "<p>You are browsing the Snappy <code>%s</code> "
+                     "documentation.</p>"
+                     "<p><a href=\"/snappy/guides/current/%s\">"
+                     "Back to the latest stable release &rsaquo;"
+                     "</a></p></div>\n"
+                     "<div class=\"eight-col\">\n") % (self.release_alias,
+                                                       self.slug, )
+            self.html = self.html.replace(before, after)
+
+    def publish(self):
+        if self.release_alias == "current":
+            # Add a guides/<page> redirect to guides/current/<page>
+            page = get_or_create_page(
+                self.title, full_url=self.full_url.replace('/current', ''),
+                redirect="/snappy/guides/current/%s" % (self.slug))
+            page.publish('en')
+        else:
+            self.title += " (%s)" % (self.release_alias,)
+        MarkdownFile.publish(self)
+
+
+def slugify(filename):
+    return os.path.basename(filename).replace('.md', '')
+
+
+def get_branch_from_lp(origin, alias):
+    return subprocess.call([
+        'bzr', 'checkout', '--lightweight', origin, alias])
+
+
+class LocalBranch:
+    titles = {}
+
+    def __init__(self, dirname, external_branch):
+        self.dirname = dirname
+        self.docs_path = os.path.join(self.dirname, DOCS_DIRNAME)
+        self.doc_fns = glob.glob(self.docs_path+'/*.md')
+        self.md_files = []
+        self.external_branch = external_branch
+        self.docs_namespace = self.external_branch.docs_namespace
+        self.release_alias = os.path.basename(self.docs_namespace)
+        self.index_doc_title = self.release_alias.title()
+        self.index_doc = self.external_branch.index_doc
+        self.markdown_class = MarkdownFile
+
+    def import_markdown(self):
+        for doc_fn in self.doc_fns:
+            if self.index_doc and os.path.basename(doc_fn) == self.index_doc:
+                md_file = self.markdown_class(
+                    doc_fn,
+                    os.path.dirname(self.docs_namespace),
+                    slug_override=os.path.basename(self.docs_namespace))
+                self.md_files.insert(0, md_file)
+            else:
+                md_file = self.markdown_class(doc_fn, self.docs_namespace)
+                self.md_files += [md_file]
+            self.titles[md_file.fn] = md_file.title
+        if not self.index_doc:
+            self._create_fake_index_doc()
+
+    def remove_old_pages(self):
+        imported_page_urls = set([md_file.full_url
+                                  for md_file in self.md_files])
+        index_doc = page_resolver.get_page_queryset_from_path(
+            self.docs_namespace)[0]
+        # All pages in this namespace currently in the database
+        db_pages = index_doc.get_descendants().all()
+        delete_pages = []
+        for db_page in db_pages:
+            still_relevant = False
+            for url in imported_page_urls:
+                if url in db_page.get_absolute_url():
+                    still_relevant = True
+                    break
+            # At this point we know that there's no match and the page
+            # can be deleted.
+            if not still_relevant:
+                delete_pages += [db_page.id]
+        # Only remove pages created by a script!
+        Page.objects.filter(id__in=delete_pages, created_by="script").delete()
+
+    def publish(self):
+        for md_file in self.md_files:
+            md_file.publish()
+
+    def _create_fake_index_doc(self):
+        '''Creates a fake index page at the top of the branches
+           docs namespace.'''
+
+        if self.docs_namespace == "current":
+            redirect = "/snappy/guides"
+        else:
+            redirect = None
+
+        in_navigation = False
+        menu_title = None
+        list_pages = ""
+        for page in self.md_files:
+            list_pages += "<li><a href=\"%s\">%s</a></li>" \
+                % (os.path.basename(page.full_url), page.title)
+        landing = (
+            u"<div class=\"row\"><div class=\"eight-col\">\n"
+            "<p>This section contains documentation for the "
+            "<code>%s</code> Snappy branch.</p>"
+            "<p><ul class=\"list-ubuntu\">%s</ul></p>\n"
+            "<p>Auto-imported from <a "
+            "href=\"https://code.launchpad.net/snappy\">%s</a>.</p>\n"
+            "</div></div>") % (self.release_alias, list_pages,
+                               self.external_branch.lp_origin)
+        new_release_page = get_or_create_page(
+            self.index_doc_title, full_url=self.docs_namespace,
+            in_navigation=in_navigation, redirect=redirect, html=landing,
+            menu_title=menu_title)
+        new_release_page.publish('en')
+
+
+class SnappyLocalBranch(LocalBranch):
+    def __init__(self, dirname, external_branch):
+        LocalBranch.__init__(self, dirname, external_branch)
+        self.markdown_class = SnappyMarkdownFile
+        self.index_doc_title = 'Snappy documentation'
+        if self.release_alias != 'current':
+            self.index_doc_title += ' (%s)' % self.release_alias
+
+    def import_markdown(self):
+        LocalBranch.import_markdown(self)
+        for md_file in self.md_files:
+            md_file.replace_links(self.titles)
+
+
+def get_or_create_page(title, full_url, menu_title=None,
+                       in_navigation=True, redirect=None, html=None):
+    # First check if pages already exist.
+    pages = Title.objects.select_related('page').filter(path__regex=full_url)
+    if pages:
+        page = pages[0].page
+        page.title = title
+        page.publisher_is_draft = True
+        page.menu_title = menu_title
+        page.in_navigation = in_navigation
+        page.redirect = redirect
+        if html:
+            # We create the page, so we know there's just one placeholder
+            placeholder = page.placeholders.all()[0]
+            if placeholder.get_plugins():
+                plugin = placeholder.get_plugins()[0].get_plugin_instance()[0]
+                plugin.body = html
+                plugin.save()
+            else:
+                add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=html)
+    else:
+        parent_pages = Title.objects.select_related('page').filter(
+            path__regex=os.path.dirname(full_url))
+        if not parent_pages:
+            print('Parent %s not found.' % os.path.dirname(full_url))
+            sys.exit(1)
+        parent = parent_pages[0].page
+
+        slug = os.path.basename(full_url)
+        page = create_page(
+            title, "default.html", "en", slug=slug, parent=parent,
+            menu_title=menu_title, in_navigation=in_navigation,
+            position="last-child", redirect=redirect)
+        if html:
+            placeholder = page.placeholders.get()
+            add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=html)
+    return page
+
+
+def import_branches(selection):
+    if not ExternalDocsBranch.objects.count():
+        logging.error('No branches registered in the '
+                      'ExternalDocsBranch table yet.')
+        return
+    tempdir = tempfile.mkdtemp()
+    for branch in ExternalDocsBranch.objects.filter(
+            docs_namespace__regex=selection):
+        checkout_location = os.path.join(
+            tempdir, os.path.basename(branch.docs_namespace))
+        if get_branch_from_lp(branch.lp_origin, checkout_location) != 0:
+            logging.error(
+                'Could not check out branch "%s".' % branch.lp_origin)
+            shutil.rmtree(checkout_location)
+            break
+        if branch.lp_origin.startswith('lp:snappy'):
+            local_branch = SnappyLocalBranch(checkout_location, branch)
+        else:
+            local_branch = LocalBranch(checkout_location, branch)
+        local_branch.import_markdown()
+        local_branch.publish()
+        local_branch.remove_old_pages()
+    shutil.rmtree(tempdir)
+
+
+class Command(BaseCommand):
+    help = "Import external branches for documentation."
+
+    def handle(*args, **options):
+        logging.basicConfig(
+            level=logging.ERROR,
+            format='%(asctime)s %(levelname)-8s %(message)s',
+            datefmt='%F %T')
+        if len(args) < 2 or args[1] == "all":
+            selection = '.*'
+        else:
+            selection = args[1]
+        import_branches(selection)
 
=== added file 'developer_portal/migrations/0003_add_external_docs_branches.py'
--- developer_portal/migrations/0003_add_external_docs_branches.py	1970-01-01 00:00:00 +0000
+++ developer_portal/migrations/0003_add_external_docs_branches.py	2015-08-11 12:06:30 +0000
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'ExternalDocsBranch'
+        db.create_table(u'developer_portal_externaldocsbranch', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('lp_origin', self.gf('django.db.models.fields.CharField')(max_length=200)),
+            ('docs_namespace', self.gf('django.db.models.fields.CharField')(max_length=120)),
+            ('index_doc', self.gf('django.db.models.fields.CharField')(max_length=120, blank=True)),
+        ))
+        db.send_create_signal(u'developer_portal', ['ExternalDocsBranch'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'ExternalDocsBranch'
+        db.delete_table(u'developer_portal_externaldocsbranch')
+
+
+    models = {
+        'cms.cmsplugin': {
+            'Meta': {'object_name': 'CMSPlugin'},
+            'changed_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'creation_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'language': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
+            'level': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
+            'lft': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
+            'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.CMSPlugin']", 'null': 'True', 'blank': 'True'}),
+            'placeholder': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.Placeholder']", 'null': 'True'}),
+            'plugin_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'position': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'rght': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
+            'tree_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'})
+        },
+        'cms.placeholder': {
+            'Meta': {'object_name': 'Placeholder'},
+            'default_width': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'slot': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'})
+        },
+        u'developer_portal.externaldocsbranch': {
+            'Meta': {'object_name': 'ExternalDocsBranch'},
+            'docs_namespace': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'index_doc': ('django.db.models.fields.CharField', [], {'max_length': '120', 'blank': 'True'}),
+            'lp_origin': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        u'developer_portal.rawhtml': {
+            'Meta': {'object_name': 'RawHtml'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            u'cmsplugin_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['cms.CMSPlugin']", 'unique': 'True', 'primary_key': 'True'})
+        }
+    }
+
+    complete_apps = ['developer_portal']
 \ No newline at end of file
 
=== modified file 'developer_portal/models.py'
--- developer_portal/models.py	2015-01-19 16:29:47 +0000
+++ developer_portal/models.py	2015-08-11 12:06:30 +0000
@@ -1,10 +1,13 @@
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
 
 from cms.models import CMSPlugin
 from djangocms_text_ckeditor.html import extract_images
 from djangocms_text_ckeditor.models import AbstractText
 
+
 class RawHtml(AbstractText):
-    
+
     class Meta:
         abstract = False
 
@@ -13,3 +16,19 @@
         body = extract_images(body, self)
         self.body = body
         AbstractText.save(self, *args, **kwargs)
+
+
+class ExternalDocsBranch(models.Model):
+    lp_origin = models.CharField(
+        max_length=200,
+        help_text=_('Launchpad branch location, ie: lp:snappy/15.04'))
+    docs_namespace = models.CharField(
+        max_length=120,
+        help_text=_('Path alias we want to use for the docs, '
+                    'ie "snappy/guides/15.04" or '
+                    '"snappy/guides/latest", etc.'))
+    index_doc = models.CharField(
+        max_length=120,
+        help_text=_('File name of doc to be used as index document, '
+                    'ie "intro.md"'),
+        blank=True)
 
=== modified file 'locale/developer_portal.pot'
--- locale/developer_portal.pot	2015-05-11 20:00:59 +0000
+++ locale/developer_portal.pot	2015-08-11 12:06:30 +0000
@@ -8,7 +8,7 @@
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2015-04-24 09:17+0000\n"
+"POT-Creation-Date: 2015-07-27 14:41+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[e-mail]>\n"
@@ -21,6 +21,16 @@
 msgid "Raw HTML"
 msgstr ""
 
+#: developer_portal/models.py:24
+msgid "Launchpad branch location, ie: lp:snappy/15.04"
+msgstr ""
+
+#: developer_portal/models.py:27
+msgid ""
+"Path alias we want to use for the docs, ie \"snappy/guides/15.04\" or "
+"\"snappy/guides/latest\", etc."
+msgstr ""
+
 #: developer_portal/settings.py:187 developer_portal/settings.py:195
 msgid "English"
 msgstr ""
 
=== modified file 'requirements.txt'
--- requirements.txt	2015-02-26 15:53:30 +0000
+++ requirements.txt	2015-08-11 12:06:30 +0000
@@ -1,6 +1,8 @@
 Django==1.6.8
+Markdown==2.6.2
 South==1.0.1
 Pillow==2.6.1
+beautifulsoup4==4.4.0
 cmsplugin-zinnia==0.6
 dj-database-url==0.3.0
 django-admin-enhancer==0.1.3.1