Merge lp:~dholbach/developer-ubuntu-com/index_page into lp:~developer-ubuntu-com-dev/developer-ubuntu-com/snappy-docs-import

[Daniel Holbach]

- Add functionality to specify an (optional) index doc.
- Stop recreating articles entries in the database for every run.

[Daniel Holbach (dholbach)]

Try with these external docs branches added:

  LP Branch Docs namespace Index doc
  -----------------------------------------------------
  lp:snappy snappy/guides/devel -
  lp:snappy/15.04 snappy/guides/current -
  lp:snapcraft snappy/snapcraft intro.md

[David Callé (davidc3)]

++1

review: Approve

=== modified file 'developer_portal/admin.py'
--- developer_portal/admin.py	2015-07-28 09:13:13 +0000
+++ developer_portal/admin.py	2015-08-10 15:40:56 +0000
@@ -12,7 +12,7 @@
 
 def manual_snappy_doc_import(modeladmin, request, queryset):
     for branch in queryset:
-        call_command('import-snappy-branches', branch.path_alias)
+        call_command('import-snappy-branches', branch.docs_namespace)
     manual_snappy_doc_import.short_description = "Import selected branches"
 
 

=== modified file 'developer_portal/management/commands/import-snappy-branches.py'
--- developer_portal/management/commands/import-snappy-branches.py	2015-07-31 12:58:32 +0000
+++ developer_portal/management/commands/import-snappy-branches.py	2015-08-10 15:40:56 +0000
@@ -1,7 +1,8 @@
 from django.core.management.base import BaseCommand
 
 from cms.api import create_page, add_plugin
-from cms.models import Title
+from cms.models import Page, Title
+from cms.utils import page_resolver
 
 from bs4 import BeautifulSoup
 import codecs
@@ -12,20 +13,25 @@
 import re
 import shutil
 import subprocess
+import sys
 import tempfile
 
 from developer_portal.models import ExternalDocsBranch
 
 DOCS_DIRNAME = 'docs'
-RELEASE_PAGES = {}
 
 
 class MarkdownFile:
     html = None
 
-    def __init__(self, fn):
+    def __init__(self, fn, docs_namespace, slug_override=None):
         self.fn = fn
-        self.slug = slugify(self.fn)
+        self.docs_namespace = docs_namespace
+        if slug_override:
+            self.slug = slug_override
+        else:
+            self.slug = slugify(self.fn)
+        self.full_url = os.path.join(self.docs_namespace, self.slug)
         with codecs.open(self.fn, 'r', encoding='utf-8') as f:
             self.html = markdown.markdown(
                 f.read(),
@@ -76,20 +82,15 @@
 
     def publish(self):
         '''Publishes pages in their branch alias namespace.'''
-        from cms.api import create_page, add_plugin
-
-        page = create_page(
-            self.title, "default.html", "en", slug=self.slug,
-            menu_title=self.title, parent=RELEASE_PAGES[self.release_alias],
-            in_navigation=True, position="last-child")
-        placeholder = page.placeholders.get()
-        add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=self.html)
+        page = get_or_create_page(
+            self.title, full_url=self.full_url, menu_title=self.title,
+            html=self.html)
         page.publish('en')
 
 
 class SnappyMarkdownFile(MarkdownFile):
-    def __init__(self, fn):
-        MarkdownFile.__init__(self, fn)
+    def __init__(self, fn, docs_namespace):
+        MarkdownFile.__init__(self, fn, docs_namespace)
         self._make_snappy_mods()
 
     def _make_snappy_mods(self):
@@ -111,10 +112,8 @@
     def publish(self):
         if self.release_alias == "current":
             # Add a guides/<page> redirect to guides/current/<page>
-            page = create_page(
-                self.title, "default.html", "en",
-                slug=self.slug, parent=RELEASE_PAGES['guides_page'],
-                in_navigation=True, position="last-child",
+            page = get_or_create_page(
+                self.title, full_url=self.full_url.replace('/current', ''),
                 redirect="/snappy/guides/current/%s" % (self.slug))
             page.publish('en')
         else:
@@ -142,90 +141,126 @@
         self.external_branch = external_branch
         self.docs_namespace = self.external_branch.docs_namespace
         self.release_alias = os.path.basename(self.docs_namespace)
-        self.overview_page_title = self.release_alias.title()
+        self.index_doc_title = self.release_alias.title()
+        self.index_doc = self.external_branch.index_doc
         self.markdown_class = MarkdownFile
 
     def import_markdown(self):
         for doc_fn in self.doc_fns:
-            md_file = self.markdown_class(doc_fn)
+            if self.index_doc and os.path.basename(doc_fn) == self.index_doc:
+                md_file = self.markdown_class(
+                    doc_fn,
+                    os.path.dirname(self.docs_namespace),
+                    slug_override=os.path.basename(self.docs_namespace))
+            else:
+                md_file = self.markdown_class(doc_fn, self.docs_namespace)
             self.md_files += [md_file]
             self.titles[md_file.fn] = md_file.title
+        if not self.index_doc:
+            self._create_fake_index_doc()
+
+    def remove_old_pages(self):
+        imported_page_urls = set([md_file.full_url
+                                  for md_file in self.md_files])
+        index_doc = page_resolver.get_page_queryset_from_path(
+            self.docs_namespace)[0]
+        # All pages in this namespace currently in the database
+        db_pages = index_doc.get_descendants().all()
+        delete_pages = []
+        for db_page in db_pages:
+            still_relevant = False
+            for url in imported_page_urls:
+                if url in db_page.get_absolute_url():
+                    still_relevant = True
+                    break
+            # At this point we know that there's no match and the page
+            # can be deleted.
+            if not still_relevant:
+                delete_pages += [db_page.id]
+        # Only remove pages created by a script!
+        Page.objects.filter(id__in=delete_pages, created_by="script").delete()
 
     def publish(self):
         for md_file in self.md_files:
             md_file.publish()
 
-    def refresh_landing_page(self):
-        '''Creates a branch page at snappy/guides/<branch alias>.'''
-
-        guides_page = Title.objects.filter(
-            path="snappy/guides", published=True,
-            language="en", publisher_is_draft=True)[0]
-        RELEASE_PAGES['guides_page'] = guides_page.page
+    def _create_fake_index_doc(self):
+        '''Creates a fake index page at the top of the branches
+           docs namespace.'''
 
         if self.docs_namespace == "current":
             redirect = "/snappy/guides"
         else:
             redirect = None
-        new_release_page = create_page(
-            self.overview_page_title, "default.html", "en",
-            slug=self.docs_namespace, parent=RELEASE_PAGES['guides_page'],
-            in_navigation=False, position="last-child", redirect=redirect)
-        placeholder = new_release_page.placeholders.get()
-        landing = (u"<div class=\"row\"><div class=\"eight-col\">\n"
-                   "<p>This section contains documentation for the "
-                   "<code>%s</code> Snappy branch.</p>"
-                   "<p>Auto-imported from <a "
-                   "href=\"https://code.launchpad.net/snappy\">%s</a>.</p>\n"
-                   "</div></div>") % (self.docs_namespace,
-                                      self.external_branch.lp_origin)
-        add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=landing)
+
+        in_navigation = False
+        menu_title = None
+        landing = (
+            u"<div class=\"row\"><div class=\"eight-col\">\n"
+            "<p>This section contains documentation for the "
+            "<code>%s</code> Snappy branch.</p>"
+            "<p>Auto-imported from <a "
+            "href=\"https://code.launchpad.net/snappy\">%s</a>.</p>\n"
+            "</div></div>") % (self.docs_namespace,
+                               self.external_branch.lp_origin)
+        new_release_page = get_or_create_page(
+            self.index_doc_title, full_url=self.docs_namespace,
+            in_navigation=in_navigation, redirect=redirect, html=landing,
+            menu_title=menu_title)
         new_release_page.publish('en')
-        RELEASE_PAGES[self.release_alias] = new_release_page
+
 
 class SnappyLocalBranch(LocalBranch):
     def __init__(self, dirname, external_branch):
         LocalBranch.__init__(self, dirname, external_branch)
         self.markdown_class = SnappyMarkdownFile
-        self.overview_page_title = 'Snappy'
+        self.index_doc_title = 'Snappy'
         if self.release_alias != 'current':
-            self.overview_page_title += ' (%s)' % self.release_alias
+            self.index_doc_title += ' (%s)' % self.release_alias
 
     def import_markdown(self):
         LocalBranch.import_markdown(self)
         for md_file in self.md_files:
             md_file.replace_links(self.titles)
 
-def remove_old_pages(selection):
-    # FIXME:
-    # - we retrieve the old article somehow
-    # - then find the Raw HTML plugin and
-    # - replace the html in there
-    # - also: remove pages we don't need anymore
-    # - add new ones
-    # - make sure we can do that for different sets of docs with
-    #   different pages
-
-    '''Removes all pages in snappy/guides, created by the importer.'''
-    from cms.models import Title, Page
-
-    pages_to_remove = []
-    aliases = "|".join(
-        ExternalDocsBranch.objects.values_list('docs_namespace', flat=True))
-    if selection == "current":
-        # Select all pages that are not in other aliases paths, this allows
-        # removing existing redirections to current and current itself
-        regex = "snappy/guides/(?!%s|.*/.*)|snappy/guides/current.*" % \
-            (aliases)
+
+def get_or_create_page(title, full_url, menu_title=None,
+                       in_navigation=True, redirect=None, html=None):
+    # First check if pages already exist.
+    pages = Title.objects.select_related('page').filter(path__regex=full_url)
+    if pages:
+        page = pages[0].page
+        page.title = title
+        page.publisher_is_draft = True
+        page.menu_title = menu_title
+        page.in_navigation = in_navigation
+        page.redirect = redirect
+        if html:
+            # We create the page, so we know there's just one placeholder
+            placeholder = page.placeholders.all()[0]
+            if placeholder.get_plugins():
+                plugin = placeholder.get_plugins()[0].get_plugin_instance()[0]
+                plugin.body = html
+                plugin.save()
+            else:
+                add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=html)
     else:
-        # Select pages that are in the selected alias path
-        regex = "snappy/guides/%s.*" % (selection,)
+        parent_pages = Title.objects.select_related('page').filter(
+            path__regex=os.path.dirname(full_url))
+        if not parent_pages:
+            print('Parent %s not found.' % os.path.dirname(full_url))
+            sys.exit(1)
+        parent = parent_pages[0].page
 
-    for g in Title.objects.select_related('page__id').filter(
-            path__regex=regex):
-        pages_to_remove.append(g.page.id)
-    # Only remove pages created by a script!
-    Page.objects.filter(id__in=pages_to_remove, created_by="script").delete()
+        slug = os.path.basename(full_url)
+        page = create_page(
+            title, "default.html", "en", slug=slug, parent=parent,
+            menu_title=menu_title, in_navigation=in_navigation,
+            position="last-child", redirect=redirect)
+        if html:
+            placeholder = page.placeholders.get()
+            add_plugin(placeholder, 'RawHtmlPlugin', 'en', body=html)
+    return page
 
 
 def import_branches(selection):
@@ -233,9 +268,6 @@
         logging.error('No branches registered in the '
                       'ExternalDocsBranch table yet.')
         return
-    # FIXME: Do the removal part last. Else we might end up in situations
-    # where some code breaks and we stay in a state without articles.
-    remove_old_pages(selection)
     tempdir = tempfile.mkdtemp()
     for branch in ExternalDocsBranch.objects.filter(
             docs_namespace__regex=selection):
@@ -250,14 +282,14 @@
             local_branch = SnappyLocalBranch(checkout_location, branch)
         else:
             local_branch = LocalBranch(checkout_location, branch)
-        local_branch.refresh_landing_page()
         local_branch.import_markdown()
         local_branch.publish()
+        local_branch.remove_old_pages()
     shutil.rmtree(tempdir)
 
 
 class Command(BaseCommand):
-    help = "Import Snappy branches for documentation."
+    help = "Import external branches for documentation."
 
     def handle(*args, **options):
         logging.basicConfig(

=== modified file 'developer_portal/migrations/0003_add_external_docs_branches.py'
--- developer_portal/migrations/0003_add_external_docs_branches.py	2015-07-27 14:46:14 +0000
+++ developer_portal/migrations/0003_add_external_docs_branches.py	2015-08-10 15:40:56 +0000
@@ -13,6 +13,7 @@
             (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
             ('lp_origin', self.gf('django.db.models.fields.CharField')(max_length=200)),
             ('docs_namespace', self.gf('django.db.models.fields.CharField')(max_length=120)),
+            ('index_doc', self.gf('django.db.models.fields.CharField')(max_length=120, blank=True)),
         ))
         db.send_create_signal(u'developer_portal', ['ExternalDocsBranch'])
 
@@ -48,6 +49,7 @@
             'Meta': {'object_name': 'ExternalDocsBranch'},
             'docs_namespace': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'index_doc': ('django.db.models.fields.CharField', [], {'max_length': '120', 'blank': 'True'}),
             'lp_origin': ('django.db.models.fields.CharField', [], {'max_length': '200'})
         },
         u'developer_portal.rawhtml': {

=== modified file 'developer_portal/models.py'
--- developer_portal/models.py	2015-07-27 14:44:23 +0000
+++ developer_portal/models.py	2015-08-10 15:40:56 +0000
@@ -25,4 +25,10 @@
     docs_namespace = models.CharField(
         max_length=120,
         help_text=_('Path alias we want to use for the docs, '
-                    'ie "snappy/guides/15.04" or "snappy/guides/latest", etc.'))
+                    'ie "snappy/guides/15.04" or '
+                    '"snappy/guides/latest", etc.'))
+    index_doc = models.CharField(
+        max_length=120,
+        help_text=_('File name of doc to be used as index document, '
+                    'ie "intro.md"'),
+        blank=True)