Merge lp:~dholbach/developer-ubuntu-com/1568871 into lp:developer-ubuntu-com

[Daniel Holbach]

List of important changes (high level overview)
 - added tests and assertions
 - remove the implicit addition of redirects (we do this in JS now
   and will have an app in the future)
 - remove the special-casing of snappy vs non-snappy docs (it was
   brittle and we will have to build proper versioning support
   anyway)
 - one way to look up pages
 - proper distinction between drafts vs. public page objects
 - refactor all page creation/updating logic into separate
   ArticlePage class
 - make index pages inherit from ArticlePage

=== modified file 'md_importer/importer/article.py'
--- md_importer/importer/article.py	2016-03-21 13:08:22 +0000
+++ md_importer/importer/article.py	2016-04-18 15:28:31 +0000
@@ -8,10 +8,15 @@
 
 from . import (
     DEFAULT_LANG,
+    DEFAULT_TEMPLATE,
     MARKDOWN_EXTENSIONS,
     SUPPORTED_ARTICLE_TYPES,
 )
-from .publish import get_or_create_page, slugify, update_page
+from .publish import (
+    ArticlePage,
+    ParentNotFoundException,
+    slugify,
+)
 
 if sys.version_info.major == 2:
     from urlparse import urlparse
@@ -20,9 +25,10 @@
 
 
 class Article:
-    def __init__(self, fn, write_to, advertise, template):
+    def __init__(self, fn, write_to, advertise=True,
+                 template=DEFAULT_TEMPLATE):
         self.html = None
-        self.page = None
+        self.article_page = None
         self.title = ""
         self.fn = fn
         self.write_to = slugify(self.fn)
@@ -116,67 +122,23 @@
 
     def add_to_db(self):
         '''Publishes pages in their branch alias namespace.'''
-        self.page = get_or_create_page(
-            title=self.title, full_url=self.full_url, menu_title=self.title,
-            html=self.html, in_navigation=self.advertise,
-            template=self.template)
-        if not self.page:
+        try:
+            self.article_page = ArticlePage(
+                title=self.title, full_url=self.full_url,
+                menu_title=self.title, html=self.html,
+                in_navigation=self.advertise, template=self.template)
+        except ParentNotFoundException:
             return False
         self.full_url = re.sub(
             r'^\/None\/', '/{}/'.format(DEFAULT_LANG),
-            self.page.get_absolute_url())
+            self.article_page.draft.get_absolute_url())
         return True
 
     def publish(self):
         if self.links_rewritten:
-            update_page(self.page, title=self.title, full_url=self.full_url,
-                        menu_title=self.title, html=self.html,
-                        in_navigation=self.advertise, template=self.template)
-        if self.page.is_dirty(DEFAULT_LANG):
-            self.page.publish(DEFAULT_LANG)
-            if self.page.get_public_object():
-                self.page = self.page.get_public_object()
-        return self.page
-
-
-class SnappyArticle(Article):
-    release_alias = None
-
-    def read(self):
-        if not Article.read(self):
-            return False
-        matches = re.findall(r'snappy/guides/(\S+?)/\S+?',
-                             self.full_url)
-        if matches:
-            self.release_alias = matches[0]
-        self._make_snappy_mods()
-        return True
-
-    def _make_snappy_mods(self):
-        # Make sure the reader knows which documentation she is browsing
-        if self.release_alias and self.release_alias != 'current':
-            before = (u"<div class=\"row no-border\">\n"
-                      "<div class=\"eight-col\">\n")
-            after = (u"<div class=\"row no-border\">\n"
-                     "<div class=\"box pull-three three-col\">"
-                     "<p>You are browsing the Snappy <code>%s</code> "
-                     "documentation.</p>"
-                     "<p><a href=\"/snappy/guides/current/%s\">"
-                     "Back to the latest stable release &rsaquo;"
-                     "</a></p></div>\n"
-                     "<div class=\"eight-col\">\n") % (self.release_alias,
-                                                       self.slug, )
-            self.html = self.html.replace(before, after)
-
-    def add_to_db(self):
-        if self.release_alias == "current":
-            # Add a guides/<page> redirect to guides/current/<page>
-            page = get_or_create_page(
-                title=self.title,
-                full_url=self.full_url.replace('/current', ''),
-                redirect="/snappy/guides/current/{}".format(self.slug))
-            if not page:
-                return False
-        elif self.release_alias:
-            self.title += " (%s)" % (self.release_alias,)
-        return Article.add_to_db(self)
+            self.article_page.update(
+                title=self.title, full_url=self.full_url,
+                menu_title=self.title, html=self.html,
+                in_navigation=self.advertise, template=self.template)
+        self.article_page.publish()
+        return self.article_page.page

=== modified file 'md_importer/importer/process.py'
--- md_importer/importer/process.py	2016-03-08 13:07:19 +0000
+++ md_importer/importer/process.py	2016-04-18 15:28:31 +0000
@@ -1,12 +1,13 @@
-from django.core.management import call_command
-
 import datetime
+import logging
 import pytz
 import shutil
 import tempfile
 
-from md_importer.importer.repo import create_repo
+from django.core.management import call_command
 
+from md_importer.importer.publish import find_page
+from md_importer.importer.repo import Repo
 from md_importer.models import (
     ExternalDocsBranchImportDirective,
     ImportedArticle,
@@ -15,10 +16,10 @@
 
 def process_branch(branch):
     tempdir = tempfile.mkdtemp()
-    repo = create_repo(tempdir, branch.origin, branch.branch_name,
-                       branch.post_checkout_command)
+    repo = Repo(tempdir, branch.origin, branch.branch_name,
+                branch.post_checkout_command)
     if repo.get() != 0:
-        return False
+        return None
     for directive in ExternalDocsBranchImportDirective.objects.filter(
             external_docs_branch=branch):
         repo.add_directive(directive.import_from,
@@ -26,29 +27,34 @@
                            directive.advertise,
                            directive.template)
     if not repo.execute_import_directives():
-        return False
+        return None
     if not repo.publish():
-        return False
+        return None
     timestamp = datetime.datetime.now(pytz.utc)
 
     # Update data in ImportedArticle table
     for page in repo.pages:
-        if ImportedArticle.objects.filter(branch=branch, page=page).count():
+        url = page.get_absolute_url()
+        if ImportedArticle.objects.filter(branch=branch, url=url).count():
             imported_article = ImportedArticle.objects.filter(
-                branch=branch, page=page)[0]
-            imported_article.last_import = datetime.datetime.now(pytz.utc)
+                branch=branch, url=url)[0]
+            imported_article.last_import = timestamp
             imported_article.save()
         else:
-            ImportedArticle.objects.get_or_create(
+            imported_article, created = ImportedArticle.objects.get_or_create(
                 branch=branch,
-                page=page,
-                last_import=datetime.datetime.now(pytz.utc))
+                url=url,
+                last_import=timestamp)
 
     # Remove old entries
     for imported_article in ImportedArticle.objects.filter(
             branch=branch, last_import__lt=timestamp):
-        imported_article.page.delete()
-        imported_article.delete()
+        page = find_page(imported_article.url)
+        if not page:
+            logging.error('Page {} not found for deletion.'.format(url))
+        else:
+            page.delete()
+            imported_article.delete()
 
     # The import is done, now let's clean up.
     imported_page_ids = [p.id for p in repo.pages
@@ -59,4 +65,4 @@
 
     # https://stackoverflow.com/questions/33284171/
     call_command('cms', 'fix-tree')
-    return True
+    return repo

=== modified file 'md_importer/importer/publish.py'
--- md_importer/importer/publish.py	2016-04-04 14:07:27 +0000
+++ md_importer/importer/publish.py	2016-04-18 15:28:31 +0000
@@ -5,7 +5,7 @@
 from md_importer.importer.tools import remove_leading_and_trailing_slash
 
 from cms.api import create_page, add_plugin
-from cms.models import Page, Title
+from cms.models import Page
 from cms.utils.page_resolver import get_page_from_path
 from djangocms_text_ckeditor.html import clean_html
 
@@ -15,6 +15,118 @@
 import os
 
 
+class ParentNotFoundException(Exception):
+    def __init__(self, parent_url):
+        self.parent_url = parent_url
+
+    def __str__(self):
+        return repr(self.parent_url)
+
+
+class ArticlePage:
+    def _text_plugin_needs_update(self, html):
+        if _compare_html(html, self.draft_text_plugin.body):
+            return False
+        if self.text_plugin and _compare_html(html, self.text_plugin.body):
+            return False
+        return True
+
+    def update(self, title, full_url, menu_title=None, in_navigation=True,
+               html=None, template=None):
+        if self.draft.get_title() != title:
+            self.draft.title = title
+        if self.draft.get_menu_title() != menu_title:
+            self.draft.menu_title = menu_title
+        if self.draft.in_navigation != in_navigation:
+            self.draft.in_navigation = in_navigation
+        if self.draft.template != template:
+            self.draft.template = template
+        if html:
+            (self.draft_placeholder,
+             self.draft_text_plugin) = get_text_plugin(self.draft)
+            if self.page:
+                (self.placeholder,
+                 self.text_plugin) = get_text_plugin(self.page)
+            if self.draft_text_plugin:
+                if self._text_plugin_needs_update(html):
+                    self.draft_text_plugin.body = html
+                    self.draft_text_plugin.save()
+                else:
+                    # Reset draft
+                    self.draft.revert(DEFAULT_LANG)
+            else:
+                self.draft_plugin = add_plugin(
+                    self.draft_placeholder, 'RawHtmlPlugin', DEFAULT_LANG,
+                    body=html)
+
+    def __init__(self, title, full_url, menu_title=None, in_navigation=True,
+                 html=None, template=DEFAULT_TEMPLATE):
+        self.page = None
+        self.draft = None
+        self.draft_placeholder = None
+        self.draft_text_plugin = None
+        self.text_plugin = None
+        self.placeholder = None
+        self.full_url = full_url
+        self.title = title
+        self.menu_title = menu_title
+        self.in_navigation = in_navigation
+        self.template = template
+
+        # First check if pages already exist.
+        self.draft = find_page(full_url, draft=True)
+        if not self.draft:
+            parent = _find_parent(full_url)
+            if not parent:
+                raise ParentNotFoundException(
+                    'Parent for {} not found.'.format(full_url))
+            slug = os.path.basename(full_url)
+            self.draft = create_page(
+                title=title, template=template, language=DEFAULT_LANG,
+                slug=slug, parent=parent, menu_title=menu_title,
+                in_navigation=in_navigation, position='last-child')
+        self.update(title, full_url, menu_title, in_navigation, html,
+                    template)
+
+    def publish(self):
+        if self.draft.is_dirty(DEFAULT_LANG):
+            self.draft.publish(DEFAULT_LANG)
+        if self.draft.get_public_object():
+            self.page = self.draft.get_public_object()
+
+
+class IndexPage(ArticlePage):
+    def __init__(self, title, full_url, menu_title='Overview',
+                 in_navigation=True, html='', template=DEFAULT_TEMPLATE):
+        self.imported_articles = []
+        self.origin = ''
+        ArticlePage.__init__(self, title, full_url, menu_title,
+                             in_navigation, html, template)
+        self.publish()
+
+    def add_imported_articles(self, imported_articles, origin):
+        self.imported_articles = imported_articles
+        self.origin = origin
+        list_pages = u''
+        for article in [a for a
+                        in self.imported_articles
+                        if a.full_url.startswith(self.full_url)]:
+            list_pages += u'<li><a href=\"{}\">{}</a></li>'.format(
+                unicode(os.path.basename(article.full_url)),
+                article.title)
+        html = (
+            u'<div class=\"row\"><div class=\"eight-col\">\n'
+            '<p>This section contains documentation for the '
+            'Snappy project.</p>'
+            '<p><ul class=\"list-ubuntu\">{}</ul></p>\n'
+            '<p>Auto-imported from <a '
+            'href=\"{}\">{}</a>.</p>\n'
+            '</div></div>'.format(list_pages, self.origin, self.origin))
+        self.update(self.title, self.full_url, self.menu_title,
+                    self.in_navigation, html, self.template)
+        self.publish()
+
+
 def _compare_html(html_a, html_b):
     soup_a = BeautifulSoup(html_a, 'html5lib')
     soup_b = BeautifulSoup(html_b, 'html5lib')
@@ -25,17 +137,28 @@
     return os.path.basename(filename).replace('.md', '').replace('.html', '')
 
 
+def clean_url(url):
+    return remove_leading_and_trailing_slash(
+        re.sub(
+            r'^\/None|{}\/'.format(DEFAULT_LANG),
+            '',
+            url))
+
+
+def find_page(url, draft=False):
+    page = get_page_from_path(clean_url(url), draft)
+    if page and draft and not page.publisher_is_draft:
+        return page.get_draft_object()
+    return page
+
+
 def _find_parent(full_url):
-    parent_url = remove_leading_and_trailing_slash(re.sub(
-        r'^\/None|{}\/'.format(DEFAULT_LANG),
-        '',
-        os.path.dirname(full_url)))
     parent_url = os.path.dirname(full_url)
     if not parent_url:
         root = Page.objects.get_home()
         if not root:
             return None
-        return root.get_draft_object()
+        return root
     parent = get_page_from_path(parent_url, draft=True)
     if not parent:
         logging.error('Parent {} not found.'.format(parent_url))
@@ -43,72 +166,18 @@
     return parent
 
 
-def find_text_plugin(page):
+def get_text_plugin(page):
+    '''Finds text plugin, creates it if necessary.'''
+    if not page:
+        return (None, None)
+    placeholders = page.placeholders.all()
+    if not placeholders:
+        return (None, None)
     # We create the page, so we know there's just one placeholder
-    placeholder = page.placeholders.all()[0]
-    if placeholder.get_plugins():
-        return (
-            placeholder,
-            placeholder.get_plugins()[0].get_plugin_instance()[0]
-        )
-    return (placeholder, None)
-
-
-def update_page(page, title, full_url, menu_title=None,
-                in_navigation=True, redirect=None, html=None, template=None):
-    if page.get_title() != title:
-        page.title = title
-    if page.get_menu_title() != menu_title:
-        page.menu_title = menu_title
-    if page.in_navigation != in_navigation:
-        page.in_navigation = in_navigation
-    if page.get_redirect() != redirect:
-        page.redirect = redirect
-    if page.template != template:
-        page.template = template
-    if html:
-        update = True
-        (placeholder, plugin) = find_text_plugin(page)
-        if plugin:
-            if _compare_html(html, plugin.body):
-                update = False
-            elif page.get_public_object():
-                (dummy, published_plugin) = \
-                    find_text_plugin(page.get_public_object())
-                if published_plugin:
-                    if _compare_html(html, published_plugin.body):
-                        update = False
-            if update:
-                plugin.body = html
-                plugin.save()
-            else:
-                # Reset draft
-                page.get_draft_object().revert(DEFAULT_LANG)
-        else:
-            add_plugin(
-                placeholder, 'RawHtmlPlugin',
-                DEFAULT_LANG, body=html)
-
-
-def get_or_create_page(title, full_url, menu_title=None,
-                       in_navigation=True, redirect=None, html=None,
-                       template=DEFAULT_TEMPLATE):
-    # First check if pages already exist.
-    pages = Title.objects.select_related('page').filter(
-        path__regex=full_url).filter(publisher_is_draft=True)
-    if pages:
-        page = pages[0].page
-        update_page(page, title, full_url, menu_title, in_navigation,
-                    redirect, html, template)
-    else:
-        parent = _find_parent(full_url)
-        if not parent:
-            return None
-        slug = os.path.basename(full_url)
-        page = create_page(
-            title=title, template=template, language=DEFAULT_LANG, slug=slug,
-            parent=parent, menu_title=menu_title, in_navigation=in_navigation,
-            position='last-child', redirect=redirect)
-        placeholder = page.placeholders.all()[0]
-        add_plugin(placeholder, 'RawHtmlPlugin', DEFAULT_LANG, body=html)
-    return page
+    plugins = placeholders[0].get_plugins()
+    if plugins:
+        return (placeholders[0], plugins[0].get_plugin_instance()[0])
+    plugin = add_plugin(
+        placeholders[0], 'RawHtmlPlugin', DEFAULT_LANG,
+        body='')
+    return (placeholders[0], plugin)

=== modified file 'md_importer/importer/repo.py'
--- md_importer/importer/repo.py	2016-04-04 08:51:39 +0000
+++ md_importer/importer/repo.py	2016-04-18 15:28:31 +0000
@@ -1,9 +1,13 @@
 from . import (
-    DEFAULT_LANG,
     SUPPORTED_ARTICLE_TYPES,
-)
-from .article import Article, SnappyArticle
-from .publish import get_or_create_page, slugify
+    DEFAULT_TEMPLATE,
+)
+from .article import Article
+from .publish import (
+    IndexPage,
+    ParentNotFoundException,
+    slugify,
+)
 from .source import SourceCode
 
 from md_importer.models import ExternalDocsBranchImportDirective
@@ -13,40 +17,22 @@
 import os
 
 
-def create_repo(tempdir, origin, branch_name, post_checkout_command):
-    if os.path.exists(origin):
-        if 'snappy' in origin:
-            repo_class = SnappyRepo
-        else:
-            repo_class = Repo
-    else:
-        if origin.startswith('lp:snappy') or \
-           'snappy' in origin.split(':')[1].split('.git')[0].split('/'):
-            repo_class = SnappyRepo
-        else:
-            repo_class = Repo
-    return repo_class(tempdir, origin, branch_name, post_checkout_command)
-
-
 class Repo:
     def __init__(self, tempdir, origin, branch_name, post_checkout_command):
         self.directives = []
         self.imported_articles = []
         self.url_map = {}
         self.titles = {}
-        self.index_doc_url = None
         self.index_page = None
-        self.release_alias = None
         # On top of the pages in imported_articles this also
         # includes index_page
         self.pages = []
+        self.urls = []
         self.origin = origin
         self.branch_name = branch_name
         self.post_checkout_command = post_checkout_command
-        branch_nick = os.path.basename(self.origin.replace('.git', ''))
-        self.checkout_location = os.path.join(
-            tempdir, branch_nick)
-        self.index_doc_title = branch_nick
+        self.branch_nick = os.path.basename(self.origin.replace('.git', ''))
+        self.checkout_location = os.path.join(tempdir, self.branch_nick)
         self.article_class = Article
 
     def get(self):
@@ -99,12 +85,20 @@
                     ]
             # If we import into a namespace and don't have an index doc,
             # we need to write one.
-            if directive['write_to'] not in [x[1] for x in import_list]:
-                self.index_doc_url = directive['write_to']
-        if self.index_doc_url:
-            if not self._create_fake_index_page():
-                logging.error('Importing of {} aborted.'.format(self.origin))
-                return False
+            # XXX: Right now we just create one index doc, this will change in
+            # the near future.
+            if directive['write_to'] not in [x[1] for x in import_list] and \
+               not self.index_page:
+                try:
+                    self.index_page = IndexPage(
+                        title=self.branch_nick, full_url=directive['write_to'],
+                        in_navigation=True, html='', menu_title=None,
+                        template=DEFAULT_TEMPLATE)
+                    self.pages.extend([self.index_page.page])
+                except ParentNotFoundException:
+                    return self._abort_import(
+                        'Could not create fake index page at {}'.format(
+                            directive['write_to']))
         # The actual import
         for entry in import_list:
             article = self._read_article(
@@ -117,12 +111,17 @@
                 # In this case the article was supported but still reading
                 # it failed, importing should be stopped here to avoid
                 # problems.
-                logging.error('Importing of {} aborted.'.format(self.origin))
-                return False
-        if self.index_doc_url:
-            self._write_fake_index_doc()
+                return self._abort_import('Reading {} failed'.format(
+                    entry[0]))
+        if self.index_page:
+            self.index_page.add_imported_articles(
+                self.imported_articles, self.origin)
         return True
 
+    def _abort_import(self, msg):
+        logging.error('Importing of {} aborted: {}.'.format(self.origin, msg))
+        return False
+
     def _read_article(self, fn, write_to, advertise, template):
         article = self.article_class(fn, write_to, advertise, template)
         if article.read():
@@ -136,56 +135,17 @@
                 return False
             article.replace_links(self.titles, self.url_map)
         for article in self.imported_articles:
-            self.pages.extend([article.publish()])
+            page = article.publish()
+            self.pages.extend([page])
+            self.urls.extend([page.get_absolute_url()])
         if self.index_page:
-            self.index_page.publish(DEFAULT_LANG)
-            self.pages.extend([self.index_page])
-        return True
-
-    def _create_fake_index_page(self):
-        '''Creates a fake index page at the top of the branches
-           docs namespace.'''
-
-        if self.index_doc_url.endswith('current'):
-            redirect = '/snappy/guides'
-        else:
-            redirect = None
-        self.index_page = get_or_create_page(
-            title=self.index_doc_title, full_url=self.index_doc_url,
-            in_navigation=False, redirect=redirect, html='',
-            menu_title=None)
-        if not self.index_page:
-            return False
-        return True
-
-    def _write_fake_index_doc(self):
-        list_pages = u''
-        for article in [a for a
-                        in self.imported_articles
-                        if a.full_url.startswith(self.index_doc_url)]:
-            list_pages += u'<li><a href=\"{}\">{}</a></li>'.format(
-                unicode(os.path.basename(article.full_url)),
-                article.title)
-        self.index_page.html = (
-            u'<div class=\"row\"><div class=\"eight-col\">\n'
-            '<p>This section contains documentation for the '
-            '<code>{}</code> Snappy branch.</p>'
-            '<p><ul class=\"list-ubuntu\">{}</ul></p>\n'
-            '<p>Auto-imported from <a '
-            'href=\"{}\">{}</a>.</p>\n'
-            '</div></div>'.format(self.release_alias, list_pages,
-                                  self.origin, self.origin))
-
-
-class SnappyRepo(Repo):
-    def __init__(self, tempdir, origin, branch_name, post_checkout_command):
-        Repo.__init__(self, tempdir, origin, branch_name,
-                      post_checkout_command)
-        self.article_class = SnappyArticle
-        self.index_doc_title = 'Snappy documentation'
-
-    def _create_fake_index_page(self):
-        self.release_alias = os.path.basename(self.index_doc_url)
-        if not self.index_doc_url.endswith('current'):
-            self.index_doc_title += ' ({})'.format(self.release_alias)
-        return Repo._create_fake_index_page(self)
+            self.urls.extend([self.index_page.page.get_absolute_url()])
+        return True
+
+    def assert_is_published(self):
+        for page in self.pages:
+            assert page.publisher_is_draft is False
+        return True
+
+    def contains_page(self, url):
+        return url in self.urls

=== modified file 'md_importer/management/commands/import_md.py'
--- md_importer/management/commands/import_md.py	2016-01-19 00:11:59 +0000
+++ md_importer/management/commands/import_md.py	2016-04-18 15:28:31 +0000
@@ -13,7 +13,7 @@
         return
     for branch in ExternalDocsBranch.objects.filter(
             origin__regex=selection, active=True):
-        if not process_branch(branch):
+        if process_branch(branch) is None:
             break
 
 

=== added file 'md_importer/migrations/0003_add_herotour_template.py'
--- md_importer/migrations/0003_add_herotour_template.py	1970-01-01 00:00:00 +0000
+++ md_importer/migrations/0003_add_herotour_template.py	2016-04-18 15:28:31 +0000
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('md_importer', '0002_hero_tour_changes'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='externaldocsbranchimportdirective',
+            name='template',
+            field=models.CharField(default=b'default.html', help_text='Django CMS template to use for the imported articles. Default: default.html', max_length=50, choices=[(b'default.html', b'Default'), (b'landing_page.html', b'Landing Page'), (b'no_subnav.html', b'Without Subnav'), (b'with_hero.html', b'With Hero'), (b'snappy_hero_tour.html', b'Snappy Hero Tour')]),
+        ),
+    ]

=== added file 'md_importer/migrations/0004_move_from_page_object_to_url_string.py'
--- md_importer/migrations/0004_move_from_page_object_to_url_string.py	1970-01-01 00:00:00 +0000
+++ md_importer/migrations/0004_move_from_page_object_to_url_string.py	2016-04-18 15:28:31 +0000
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('md_importer', '0003_add_herotour_template'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='importedarticle',
+            name='page',
+        ),
+        migrations.AddField(
+            model_name='importedarticle',
+            name='url',
+            field=models.CharField(default='imported', help_text='URL of article, e.g. snappy/guides/security', max_length=300),
+            preserve_default=False,
+        ),
+    ]

=== modified file 'md_importer/models.py'
--- md_importer/models.py	2016-03-08 09:31:07 +0000
+++ md_importer/models.py	2016-04-18 15:28:31 +0000
@@ -2,8 +2,6 @@
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
-from cms.models import Page
-
 
 if settings.CMS_TEMPLATES:
     cms_templates = settings.CMS_TEMPLATES
@@ -71,7 +69,14 @@
 
 
 class ImportedArticle(models.Model):
-    page = models.ForeignKey(Page)
+    url = models.CharField(
+        max_length=300,
+        help_text=_('URL of article, e.g. snappy/guides/security'),
+    )
     branch = models.ForeignKey(ExternalDocsBranch)
     last_import = models.DateTimeField(
         _('Datetime'), help_text=_('Datetime of last import.'))
+
+    def __str__(self):
+        return '{} -- {} -- {}'.format(
+            self.url, self.branch, self.last_import)

=== added file 'md_importer/tests/test_article.py'
--- md_importer/tests/test_article.py	1970-01-01 00:00:00 +0000
+++ md_importer/tests/test_article.py	2016-04-18 15:28:31 +0000
@@ -0,0 +1,155 @@
+import os
+
+from django.test import TestCase
+
+from md_importer.importer.article import Article
+
+from .utils import (
+    db_create_root_page,
+    db_empty_page_list,
+    PublishedPages,
+)
+
+
+class TestArticleCreate(TestCase):
+    def runTest(self):
+        fn = os.path.join(
+            os.path.dirname(__file__),
+            'data/snapcraft-test/docs/plugins.md')
+        article = Article(fn, 'plugins')
+        self.assertTrue(article.read())
+
+
+class TestArticlePublish(TestCase):
+    def runTest(self):
+        db_empty_page_list()
+        db_create_root_page()
+        fn = os.path.join(
+            os.path.dirname(__file__),
+            'data/snapcraft-test/docs/plugins.md')
+        article = Article(fn, 'plugins')
+        self.assertTrue(article.read())
+        self.assertTrue(article.add_to_db())
+        self.assertTrue(article.publish())
+
+
+class TestArticlePublishTwiceNoHTMLChange(TestCase):
+    def runTest(self):
+        db_empty_page_list()
+        db_create_root_page()
+        fn = os.path.join(
+            os.path.dirname(__file__),
+            'data/snapcraft-test/docs/plugins.md')
+        article = Article(fn, 'plugins')
+        self.assertTrue(article.read())
+        self.assertTrue(article.add_to_db())
+        self.assertTrue(article.publish())
+        html = article.html
+        self.assertGreater(
+            len(html), len(open(fn).read()))
+        self.assertTrue(article.publish())
+        self.assertEqual(html, article.html)
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(1+1))  # Root + article page
+
+
+class TestArticleImportTwiceNoHTMLChange(TestCase):
+    def runTest(self):
+        db_empty_page_list()
+        db_create_root_page()
+        fn = os.path.join(
+            os.path.dirname(__file__),
+            'data/snapcraft-test/docs/plugins.md')
+        article = Article(fn, 'plugins')
+        self.assertTrue(article.read())
+        self.assertTrue(article.add_to_db())
+        self.assertTrue(article.publish())
+        html = article.html
+        self.assertGreater(
+            len(html), len(open(fn).read()))
+        article = Article(fn, 'plugins')
+        self.assertTrue(article.read())
+        self.assertTrue(article.add_to_db())
+        self.assertTrue(article.publish())
+        self.assertEqual(html, article.html)
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(1+1))  # Root + article page
+
+
+class TestArticleLinkRewrite(TestCase):
+    def runTest(self):
+        db_empty_page_list()
+        db_create_root_page()
+        path = os.path.join(
+            os.path.dirname(__file__), 'data/snapcraft-test/docs/')
+
+        # Read first article
+        intro_fn = os.path.join(path, 'intro.md')
+        intro = Article(intro_fn, 'intro')
+        self.assertTrue(intro.read())
+        self.assertTrue(intro.add_to_db())
+        html = intro.html
+
+        # Read second article
+        get_started_fn = os.path.join(path, 'get-started.md')
+        get_started = Article(get_started_fn, 'get-started')
+        self.assertTrue(get_started.read())
+        self.assertTrue(get_started.add_to_db())
+
+        # Replace just one link: [get set up](get-started.md)
+        intro.replace_links(
+            {get_started_fn: 'get set up'},
+            {get_started_fn: get_started})
+        self.assertNotEqual(html, intro.html)
+        self.assertTrue(intro.publish())
+
+
+class TestArticleLinkRewriteTwice(TestCase):
+    def runTest(self):
+        db_empty_page_list()
+        db_create_root_page()
+        path = os.path.join(
+            os.path.dirname(__file__), 'data/snapcraft-test/docs/')
+
+        # Read first article
+        intro_fn = os.path.join(path, 'intro.md')
+        intro = Article(intro_fn, 'intro')
+        self.assertTrue(intro.read())
+        self.assertTrue(intro.add_to_db())
+        html = intro.html
+
+        # Read second article
+        get_started_fn = os.path.join(path, 'get-started.md')
+        get_started = Article(get_started_fn, 'get-started')
+        self.assertTrue(get_started.read())
+        self.assertTrue(get_started.add_to_db())
+
+        # Replace just one link: [get set up](get-started.md)
+        intro.replace_links(
+            {get_started_fn: 'get set up'},
+            {get_started_fn: get_started})
+        # Check if the HTML changed because of replaced links
+        self.assertNotEqual(html, intro.html)
+        self.assertTrue(intro.publish())
+
+        # Now check if the HTML will change after a second run
+        html = intro.html
+
+        # 2nd time: Read first article
+        intro = Article(intro_fn, 'intro')
+        self.assertTrue(intro.read())
+        self.assertTrue(intro.add_to_db())
+
+        # 2nd time: Read second article
+        get_started = Article(get_started_fn, 'get-started')
+        self.assertTrue(get_started.read())
+        self.assertTrue(get_started.add_to_db())
+
+        # 2nd time: Replace just one link: [get set up](get-started.md)
+        intro.replace_links(
+            {get_started_fn: 'get set up'},
+            {get_started_fn: get_started})
+        self.assertTrue(intro.publish())
+
+        # After the second run, the HTML should still be the same
+        self.assertEqual(html, intro.html)

=== modified file 'md_importer/tests/test_branch_fetch.py'
--- md_importer/tests/test_branch_fetch.py	2016-01-08 11:02:05 +0000
+++ md_importer/tests/test_branch_fetch.py	2016-04-18 15:28:31 +0000
@@ -4,13 +4,13 @@
 
 from django.test import TestCase
 
-from md_importer.importer.repo import create_repo, Repo
+from md_importer.importer.repo import Repo
 
 
 class TestBranchFetch(TestCase):
     def test_git_fetch(self):
         tempdir = tempfile.mkdtemp()
-        repo = create_repo(
+        repo = Repo(
             tempdir,
             'https://github.com/ubuntu-core/snapcraft.git',
             'master',
@@ -22,7 +22,7 @@
 
     def test_bzr_fetch(self):
         tempdir = tempfile.mkdtemp()
-        repo = create_repo(
+        repo = Repo(
             tempdir,
             'lp:snapcraft',  # outdated, but should work for testing
             '',
@@ -34,7 +34,7 @@
 
     def test_post_checkout_command(self):
         tempdir = tempfile.mkdtemp()
-        repo = create_repo(
+        repo = Repo(
             tempdir,
             'lp:snapcraft',
             '',

=== modified file 'md_importer/tests/test_branch_import.py'
--- md_importer/tests/test_branch_import.py	2016-04-04 13:55:09 +0000
+++ md_importer/tests/test_branch_import.py	2016-04-18 15:28:31 +0000
@@ -10,10 +10,13 @@
     TEMPLATE_CHOICES,
 )
 from md_importer.importer.article import Article
-from md_importer.importer.publish import find_text_plugin
+from md_importer.importer.publish import get_text_plugin
 from md_importer.importer.tools import remove_trailing_slash
 
-from .utils import TestLocalBranchImport
+from .utils import (
+    PublishedPages,
+    TestLocalBranchImport,
+)
 
 
 class TestImportDirectivesBuildHierarchyImport(TestLocalBranchImport):
@@ -50,8 +53,9 @@
         # Make sure we got the parents right
         for article in self.repo.imported_articles:
             parent_url = remove_trailing_slash(
-                article.page.parent.get_absolute_url())
-            url = remove_trailing_slash(article.page.get_absolute_url())
+                article.article_page.page.parent.get_absolute_url())
+            url = remove_trailing_slash(
+                article.article_page.page.get_absolute_url())
             self.assertEqual(parent_url, os.path.split(url)[0])
             self.assertIsInstance(article, Article)
 
@@ -62,10 +66,11 @@
         self.repo.add_directive('docs', '')
         self.assertEqual(len(self.repo.directives), 1)
         self.assertTrue(self.repo.execute_import_directives())
-        self.assertGreater(len(self.repo.imported_articles), 3)
+        self.assertGreater(len(self.repo.imported_articles), 10)
         self.assertTrue(self.repo.publish())
-        pages = Page.objects.all()
-        self.assertGreater(pages.count(), len(self.repo.imported_articles))
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
+            len(self.repo.imported_articles)+1))  # + landing/index page
         for article in self.repo.imported_articles:
             self.assertIsInstance(article, Article)
 
@@ -78,13 +83,14 @@
         self.repo.add_directive('HACKING.md', 'hacking')
         self.assertEqual(len(self.repo.directives), 3)
         self.assertTrue(self.repo.execute_import_directives())
-        self.assertGreater(len(self.repo.imported_articles), 5)
+        self.assertGreater(len(self.repo.imported_articles), 10)
         self.assertTrue(self.repo.publish())
-        pages = Page.objects.filter(publisher_is_draft=False)
-        self.assertEqual(pages.count(), len(self.repo.imported_articles))
-        self.assertGreater(pages.count(), 5)
-        self.assertIn(u'/en/', [p.get_absolute_url() for p in pages])
-        self.assertIn(u'/en/hacking/', [p.get_absolute_url() for p in pages])
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
+            len(self.repo.imported_articles)))
+        self.assertTrue(published_pages.has_at_least_size(10))
+        self.assertTrue(published_pages.contains_url(u'/en/'))
+        self.assertTrue(published_pages.contains_url('/en/hacking/'))
 
 
 class TestArticletreeOneFileImport(TestLocalBranchImport):
@@ -96,9 +102,10 @@
         self.assertTrue(self.repo.execute_import_directives())
         self.assertEqual(len(self.repo.imported_articles), 1)
         self.assertTrue(self.repo.publish())
-        self.assertEqual(
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
             1+1,  # readme + root
-            Page.objects.filter(publisher_is_draft=False).count())
+        ))
         self.assertEqual(self.repo.pages[0].parent, self.root)
 
 
@@ -109,7 +116,8 @@
         self.repo.add_directive('docs', '')
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
-        for page in Page.objects.filter(publisher_is_draft=False):
+        published_pages = PublishedPages()
+        for page in published_pages.pages:
             if page.parent is not None:
                 self.assertEqual(page.parent_id, self.root.id)
 
@@ -120,10 +128,11 @@
         self.repo.add_directive('docs', '')
         self.assertEqual(len(self.repo.directives), 1)
         self.assertTrue(self.repo.execute_import_directives())
-        self.assertGreater(len(self.repo.imported_articles), 3)
+        self.assertGreater(len(self.repo.imported_articles), 10)
         self.assertTrue(self.repo.publish())
-        pages = Page.objects.all()
-        self.assertGreater(pages.count(), len(self.repo.imported_articles))
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
+            len(self.repo.imported_articles)+1))  # + landing/index page
         for article in self.repo.imported_articles:
             self.assertIsInstance(article, Article)
             self.assertNotIn('<body>', article.html)
@@ -136,14 +145,15 @@
         self.repo.add_directive('docs', '')
         self.assertEqual(len(self.repo.directives), 1)
         self.assertTrue(self.repo.execute_import_directives())
-        self.assertGreater(len(self.repo.imported_articles), 3)
+        self.assertGreater(len(self.repo.imported_articles), 10)
         self.assertTrue(self.repo.publish())
-        pages = Page.objects.all()
-        self.assertGreater(pages.count(), len(self.repo.imported_articles))
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
+            len(self.repo.imported_articles)+1))  # + landing/index page
         for article in self.repo.imported_articles:
             self.assertIsInstance(article, Article)
             self.assertNotIn('/None/', article.full_url)
-        for page in pages:
+        for page in published_pages.pages:
             self.assertIsNotNone(page.get_slug())
 
 
@@ -157,7 +167,8 @@
         for article in self.repo.imported_articles:
             self.assertTrue(article.advertise)
         self.assertTrue(self.repo.publish())
-        for page in Page.objects.filter(publisher_is_draft=False):
+        published_pages = PublishedPages()
+        for page in published_pages.pages:
             if page.parent is not None:
                 parent = page.parent.get_public_object()
                 self.assertEqual(parent.id, self.root.id)
@@ -174,7 +185,8 @@
         for article in self.repo.imported_articles:
             self.assertFalse(article.advertise)
         self.assertTrue(self.repo.publish())
-        for page in Page.objects.filter(publisher_is_draft=False):
+        published_pages = PublishedPages()
+        for page in published_pages.pages:
             if page.parent is not None:
                 self.assertEqual(page.parent_id, self.root.id)
                 self.assertFalse(page.in_navigation)
@@ -188,9 +200,9 @@
         self.repo.add_directive('docs', '')
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
-        self.assertEqual(
-            Page.objects.filter(publisher_is_draft=False).count(),
-            len(self.repo.imported_articles)+1)  # articles + root
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
+            len(self.repo.imported_articles)+1))  # articles + root
         # Run second import
         shutil.rmtree(self.tempdir)
         self.create_repo('data/snapcraft-test')
@@ -199,9 +211,9 @@
         self.assertEqual(len(self.repo.imported_articles), 0)
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
-        self.assertEqual(
-            Page.objects.filter(publisher_is_draft=False).count(),
-            len(self.repo.imported_articles)+1)  # articles + root
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
+            len(self.repo.imported_articles)+1))  # articles + root
 
 
 class TestTwiceImportNoHtmlChange(TestLocalBranchImport):
@@ -212,9 +224,9 @@
         self.repo.add_directive('docs', '')
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
-        self.assertEqual(
-            Page.objects.filter(publisher_is_draft=False).count(),
-            len(self.repo.imported_articles)+1)  # articles + root
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(
+            len(self.repo.imported_articles)+1))  # articles + root
         shutil.rmtree(self.tempdir)
         # Take the time before publishing the second import
         now = datetime.now(pytz.utc)
@@ -226,9 +238,10 @@
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
         # Check the page's plugins
-        for page in Page.objects.filter(publisher_is_draft=False):
+        published_pages.update()
+        for page in published_pages.pages:
             if page != self.root:
-                (dummy, plugin) = find_text_plugin(page)
+                (dummy, plugin) = get_text_plugin(page)
                 self.assertGreater(now, plugin.changed_date)
 
 
@@ -241,7 +254,8 @@
         for article in self.repo.imported_articles:
             self.assertEqual(article.template, DEFAULT_TEMPLATE)
         self.assertTrue(self.repo.publish())
-        for page in Page.objects.filter(publisher_is_draft=False):
+        published_pages = PublishedPages()
+        for page in published_pages.pages:
             if page.parent is not None:
                 self.assertEqual(page.template, DEFAULT_TEMPLATE)
 
@@ -256,6 +270,7 @@
         for article in self.repo.imported_articles:
             self.assertEqual(article.template, template_to_use)
         self.assertTrue(self.repo.publish())
-        for page in Page.objects.filter(publisher_is_draft=False):
+        published_pages = PublishedPages()
+        for page in published_pages.pages:
             if page.parent is not None:
                 self.assertEqual(page.template, template_to_use)

=== modified file 'md_importer/tests/test_import_process.py'
--- md_importer/tests/test_import_process.py	2016-01-16 00:36:39 +0000
+++ md_importer/tests/test_import_process.py	2016-04-18 15:28:31 +0000
@@ -4,6 +4,7 @@
 from cms.api import publish_pages
 from cms.models import Page
 
+from md_importer.importer import DEFAULT_LANG
 from md_importer.importer.process import process_branch
 from md_importer.models import (
     ExternalDocsBranch,
@@ -11,9 +12,12 @@
     ImportedArticle,
 )
 from .utils import (
+    check_repo,
+    check_imported_article,
     db_add_empty_page,
     db_create_root_page,
     db_empty_page_list,
+    PublishedPages,
 )
 
 
@@ -41,7 +45,36 @@
             import_from='HACKING.md',
             write_to='snappy/build-apps/devel/hacking',
             external_docs_branch=branch)
-        self.assertTrue(process_branch(branch))
+        self.assertIsNotNone(process_branch(branch))
+
+
+class TestPageStateAfterImportProcess(TestCase):
+    def runTest(self):
+        db_empty_page_list()
+        root = db_create_root_page()
+        snappy_page = db_add_empty_page('Snappy', root)
+        build_apps = db_add_empty_page('Build Apps', snappy_page)
+        publish_pages([snappy_page, build_apps])
+        ExternalDocsBranch.objects.all().delete()
+        ExternalDocsBranchImportDirective.objects.all().delete()
+        ImportedArticle.objects.all().delete()
+        branch, created = ExternalDocsBranch.objects.get_or_create(
+            origin=os.path.join(
+                os.path.dirname(__file__), 'data/snapcraft-test'),
+            branch_name='')
+        a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='README.md', write_to='snappy/build-apps/devel',
+            external_docs_branch=branch)
+        b, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs', write_to='snappy/build-apps/devel',
+            external_docs_branch=branch)
+        c, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='HACKING.md',
+            write_to='snappy/build-apps/devel/hacking',
+            external_docs_branch=branch)
+        self.assertIsNotNone(process_branch(branch))
+        for imported_article in ImportedArticle.objects.all():
+            self.assertTrue(check_imported_article(imported_article))
 
 
 class TestImportProcessBranchWhichChangesFiles(TestCase):
@@ -57,12 +90,241 @@
             branch_name='')
         a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
             import_from='', write_to='', external_docs_branch=branch)
-        self.assertTrue(process_branch(branch))
-        self.assertEqual(
-            Page.objects.filter(publisher_is_draft=False).count(), 3)
+        self.assertIsNotNone(process_branch(branch))
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(3))
         branch.origin = os.path.join(
             os.path.dirname(__file__), 'data/link2-test')
         branch.save()
         self.assertTrue(process_branch(branch))
-        self.assertEqual(
-            Page.objects.filter(publisher_is_draft=False).count(), 3)
+        published_pages.update()
+        self.assertTrue(published_pages.has_size(3))
+
+
+class TestImportProcessTwice(TestCase):
+    '''
+    This is a pretty standard snapcraft import. snappy/build-apps are
+    already created. Running the import twice used to create problems,
+    because the snappy/build-apps used to be deleted at the end of
+    process.process_branch. This test will make sure this will never
+    happen again.
+    '''
+    def runTest(self):
+        db_empty_page_list()
+        root = db_create_root_page()
+        snappy_page = db_add_empty_page('Snappy', root)
+        build_apps = db_add_empty_page(
+            'Build Apps', snappy_page, slug='build-apps')
+        publish_pages([snappy_page, build_apps])
+        ExternalDocsBranch.objects.all().delete()
+        ExternalDocsBranchImportDirective.objects.all().delete()
+        ImportedArticle.objects.all().delete()
+        branch, created = ExternalDocsBranch.objects.get_or_create(
+            origin=os.path.join(
+                os.path.dirname(__file__), 'data/snapcraft-test'),
+            branch_name='')
+        a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs/intro.md', write_to='snappy/build-apps',
+            external_docs_branch=branch)
+        b, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs', write_to='snappy/build-apps',
+            external_docs_branch=branch)
+        repo = process_branch(branch)
+        self.assertTrue(check_repo(repo))
+        self.assertGreater(len(repo.pages), 10)
+        self.assertTrue(repo.contains_page(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.has_at_least_size(10))
+        for imported_article in ImportedArticle.objects.all():
+            self.assertTrue(check_imported_article(imported_article))
+
+        # Run the import a second time
+        repo = process_branch(branch)
+        self.assertTrue(check_repo(repo))
+        self.assertGreater(len(repo.pages), 10)
+        self.assertTrue(repo.contains_page(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        published_pages.update()
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.has_at_least_size(10))
+        for imported_article in ImportedArticle.objects.all():
+            self.assertTrue(check_imported_article(imported_article))
+
+
+class TestWholeImportProcessTwiceVariantA(TestCase):
+    '''
+    This is a pretty standard snapcraft import. snappy/build-apps are
+    already created. Running the import twice used to create problems,
+    because the snappy/build-apps used to be deleted at the end of
+    process.process_branch. This test will make sure this will never
+    happen again.
+    This time we test a whole import, ie Snappy + Snapcraft twice.
+    '''
+    def runTest(self):
+        db_empty_page_list()
+        self.assertEqual(Page.objects.count(), 0)
+        root = db_create_root_page()
+        snappy_page = db_add_empty_page('Snappy', root)
+        build_apps = db_add_empty_page(
+            'Build Apps', snappy_page, slug='build-apps')
+        guides = db_add_empty_page('Guides', snappy_page)
+        phone = db_add_empty_page('Phone', root)
+        publish_pages([snappy_page, build_apps, guides, phone])
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(5))
+        ExternalDocsBranch.objects.all().delete()
+        ExternalDocsBranchImportDirective.objects.all().delete()
+        ImportedArticle.objects.all().delete()
+
+        # Snapcraft import definition
+        snapcraft_branch, created = ExternalDocsBranch.objects.get_or_create(
+            origin=os.path.join(
+                os.path.dirname(__file__), 'data/snapcraft-test'),
+            branch_name='')
+        a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs/intro.md', write_to='snappy/build-apps',
+            external_docs_branch=snapcraft_branch)
+        b, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs', write_to='snappy/build-apps',
+            external_docs_branch=snapcraft_branch)
+
+        # Snappy import definition
+        snappy_branch, created = ExternalDocsBranch.objects.get_or_create(
+            origin=os.path.join(
+                os.path.dirname(__file__), 'data/snappy-test'),
+            branch_name='')
+        c, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs', write_to='snappy/guides',
+            external_docs_branch=snappy_branch)
+
+        # Run the import a first time
+        snapcraft_repo = process_branch(snapcraft_branch)
+        self.assertTrue(check_repo(snapcraft_repo))
+        self.assertGreater(len(snapcraft_repo.pages), 10)
+        self.assertTrue(snapcraft_repo.contains_page(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        snappy_repo = process_branch(snappy_branch)
+        self.assertTrue(check_repo(snappy_repo))
+        self.assertGreater(len(snappy_repo.pages), 10)
+        self.assertTrue(snappy_repo.contains_page(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.has_at_least_size(20))
+        for imported_article in ImportedArticle.objects.all():
+            self.assertTrue(check_imported_article(imported_article))
+
+        # Run the import a second time
+        snapcraft_repo = process_branch(snapcraft_branch)
+        self.assertTrue(check_repo(snapcraft_repo))
+        self.assertGreater(len(snapcraft_repo.pages), 10)
+        self.assertTrue(snapcraft_repo.contains_page(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        snappy_repo = process_branch(snappy_branch)
+        self.assertTrue(check_repo(snappy_repo))
+        self.assertGreater(len(snappy_repo.pages), 10)
+        self.assertTrue(snappy_repo.contains_page(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+
+        published_pages.update()
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.has_at_least_size(20))
+        for imported_article in ImportedArticle.objects.all():
+            self.assertTrue(check_imported_article(imported_article))
+
+
+class TestWholeImportProcessTwiceVariantB(TestCase):
+    '''
+    This is a pretty standard snapcraft import. snappy/build-apps are
+    already created. Running the import twice used to create problems,
+    because the snappy/build-apps used to be deleted at the end of
+    process.process_branch. This test will make sure this will never
+    happen again.
+    This time we test a whole import, ie Snappy + Snapcraft twice.
+    '''
+    def runTest(self):
+        db_empty_page_list()
+        self.assertEqual(Page.objects.count(), 0)
+        root = db_create_root_page()
+        snappy_page = db_add_empty_page('Snappy', root)
+        guides = db_add_empty_page('Guides', snappy_page)
+        publish_pages([snappy_page, guides])
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(3))
+        ExternalDocsBranch.objects.all().delete()
+        ExternalDocsBranchImportDirective.objects.all().delete()
+        ImportedArticle.objects.all().delete()
+
+        # Snapcraft import definition
+        snapcraft_branch, created = ExternalDocsBranch.objects.get_or_create(
+            origin=os.path.join(
+                os.path.dirname(__file__), 'data/snapcraft-test'),
+            branch_name='')
+        a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs/intro.md', write_to='snappy/build-apps',
+            external_docs_branch=snapcraft_branch)
+        b, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs', write_to='snappy/build-apps',
+            external_docs_branch=snapcraft_branch)
+
+        # Snappy import definition
+        snappy_branch, created = ExternalDocsBranch.objects.get_or_create(
+            origin=os.path.join(
+                os.path.dirname(__file__), 'data/snappy-test'),
+            branch_name='')
+        c, created = ExternalDocsBranchImportDirective.objects.get_or_create(
+            import_from='docs', write_to='snappy/guides',
+            external_docs_branch=snappy_branch)
+
+        # Run the import a first time
+        snapcraft_repo = process_branch(snapcraft_branch)
+        self.assertTrue(check_repo(snapcraft_repo))
+        self.assertGreater(len(snapcraft_repo.pages), 10)
+        self.assertTrue(snapcraft_repo.contains_page(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        snappy_repo = process_branch(snappy_branch)
+        self.assertTrue(check_repo(snappy_repo))
+        self.assertGreater(len(snappy_repo.pages), 10)
+        self.assertTrue(snappy_repo.contains_page(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.has_at_least_size(20))
+        for imported_article in ImportedArticle.objects.all():
+            self.assertTrue(check_imported_article(imported_article))
+
+        # Run the import a second time
+        snapcraft_repo = process_branch(snapcraft_branch)
+        self.assertTrue(check_repo(snapcraft_repo))
+        self.assertGreater(len(snapcraft_repo.pages), 10)
+        self.assertTrue(snapcraft_repo.contains_page(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        snappy_repo = process_branch(snappy_branch)
+        self.assertTrue(check_repo(snappy_repo))
+        self.assertGreater(len(snappy_repo.pages), 10)
+        self.assertTrue(snappy_repo.contains_page(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+
+        published_pages.update()
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/build-apps/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.contains_url(
+            '/{}/snappy/guides/'.format(DEFAULT_LANG)))
+        self.assertTrue(published_pages.has_at_least_size(20))
+        for imported_article in ImportedArticle.objects.all():
+            self.assertTrue(check_imported_article(imported_article))

=== modified file 'md_importer/tests/test_link_rewrite.py'
--- md_importer/tests/test_link_rewrite.py	2016-03-22 10:52:47 +0000
+++ md_importer/tests/test_link_rewrite.py	2016-04-18 15:28:31 +0000
@@ -7,6 +7,7 @@
 from .utils import (
     db_add_empty_page,
     is_local_link,
+    PublishedPages,
     TestLocalBranchImport,
 )
 
@@ -17,11 +18,11 @@
         self.repo.add_directive('', '')
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
-        pages = Page.objects.filter(publisher_is_draft=False)
-        self.assertEqual(pages.count(), 1+2)  # root + 2 articles
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(1+2))  # root + 2 articles
         for article in self.repo.imported_articles:
             self.assertTrue(isinstance(article, Article))
-            self.assertEqual(article.page.parent, self.root)
+            self.assertEqual(article.article_page.page.parent, self.root)
             soup = BeautifulSoup(article.html, 'html5lib')
             for link in soup.find_all('a'):
                 page = self.check_local_link(link.attrs['href'])
@@ -29,8 +30,8 @@
                     page,
                     msg='Link {} not found. Available pages: {}'.format(
                         link.attrs['href'],
-                        ', '.join([p.get_absolute_url() for p in pages])))
-                self.assertIn(page, pages)
+                        ', '.join(published_pages.urls)))
+                self.assertIn(page, published_pages.pages)
             if article.slug == 'file1':
                 for link in soup.find_all('a'):
                     if not link.has_attr('class') or \
@@ -46,18 +47,18 @@
         self.repo.add_directive('', '')
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
-        pages = Page.objects.filter(publisher_is_draft=False)
-        self.assertEqual(pages.count(), 1+2)  # root + 2 articles
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(1+2))  # root + 2 articles
         for article in self.repo.imported_articles:
             self.assertTrue(isinstance(article, Article))
-            self.assertEqual(article.page.parent, self.root)
+            self.assertEqual(article.article_page.page.parent, self.root)
             soup = BeautifulSoup(article.html, 'html5lib')
             for link in soup.find_all('a'):
                 if not link.has_attr('class') or \
                    'headeranchor-link' not in link.attrs['class']:
                     page = self.check_local_link(link.attrs['href'])
                     self.assertIsNone(page)
-                    self.assertNotIn(page, pages)
+                    self.assertNotIn(page, published_pages.pages)
 
 
 class TestNoneNotInLinks(TestLocalBranchImport):
@@ -67,8 +68,8 @@
         self.assertFalse(snappy_page.publisher_is_draft)
         build_apps = db_add_empty_page('Build Apps', snappy_page)
         self.assertFalse(build_apps.publisher_is_draft)
-        self.assertEqual(
-            3, Page.objects.filter(publisher_is_draft=False).count())
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(3))
         self.repo.add_directive('docs/intro.md', 'snappy/build-apps/current')
         self.repo.add_directive('docs', 'snappy/build-apps/current')
         self.assertTrue(self.repo.execute_import_directives())
@@ -89,8 +90,8 @@
         self.assertFalse(snappy_page.publisher_is_draft)
         build_apps = db_add_empty_page('Build Apps', snappy_page)
         self.assertFalse(build_apps.publisher_is_draft)
-        self.assertEqual(
-            3, Page.objects.filter(publisher_is_draft=False).count())
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(3))
         self.repo.add_directive('docs/intro.md', 'snappy/build-apps/current')
         self.repo.add_directive('docs', 'snappy/build-apps/current')
         self.assertTrue(self.repo.execute_import_directives())

=== modified file 'md_importer/tests/test_misc.py'
--- md_importer/tests/test_misc.py	2016-01-15 14:07:21 +0000
+++ md_importer/tests/test_misc.py	2016-04-18 15:28:31 +0000
@@ -1,8 +1,7 @@
 from bs4 import BeautifulSoup
 
-from cms.models import Page
-
 from .utils import (
+    PublishedPages,
     TestLocalBranchImport,
 )
 
@@ -20,10 +19,10 @@
         self.repo.add_directive('', '')
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
-        pages = Page.objects.filter(publisher_is_draft=False)
-        self.assertEqual(pages.count(), 1+1)  # root + 1 article
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(1+1))  # root + 1 article
         for article in self.repo.imported_articles:
-            self.assertEqual(article.page.parent, self.root)
+            self.assertEqual(article.article_page.page.parent, self.root)
             soup = BeautifulSoup(article.html, 'html5lib')
             for link in soup.find_all('a'):
                 page = self.check_local_link(link.attrs['href'])
@@ -31,5 +30,5 @@
                     page,
                     msg='Link {} not found. Available pages: {}'.format(
                         link.attrs['href'],
-                        ', '.join([p.get_absolute_url() for p in pages])))
-                self.assertIn(page, pages)
+                        ', '.join(published_pages.urls)))
+                self.assertIn(page, published_pages.pages)

=== modified file 'md_importer/tests/test_snappy_import.py'
--- md_importer/tests/test_snappy_import.py	2016-01-11 14:02:04 +0000
+++ md_importer/tests/test_snappy_import.py	2016-04-18 15:28:31 +0000
@@ -1,11 +1,11 @@
 from cms.api import publish_pages
 from cms.models import Page
 
-from md_importer.importer import DEFAULT_LANG
-from md_importer.importer.repo import SnappyRepo
-from md_importer.importer.article import SnappyArticle
+from md_importer.importer.repo import Repo
+from md_importer.importer.article import Article
 from .utils import (
     db_add_empty_page,
+    PublishedPages,
     TestLocalBranchImport,
 )
 
@@ -16,46 +16,16 @@
         snappy_page = db_add_empty_page('Snappy', self.root)
         guides = db_add_empty_page('Guides', snappy_page)
         publish_pages([snappy_page, guides])
-        self.assertTrue(isinstance(self.repo, SnappyRepo))
+        self.assertTrue(isinstance(self.repo, Repo))
         self.repo.add_directive('docs', 'snappy/guides/devel')
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
         for article in self.repo.imported_articles:
-            self.assertTrue(isinstance(article, SnappyArticle))
+            self.assertTrue(isinstance(article, Article))
         self.assertGreater(len(self.repo.pages), 0)
         devel = Page.objects.filter(parent=guides.get_public_object())
         self.assertEqual(devel.count(), 1)
-        for page in Page.objects.filter(publisher_is_draft=False):
+        published_pages = PublishedPages()
+        for page in published_pages.pages:
             if page not in [self.root, snappy_page, guides, devel[0]]:
                 self.assertEqual(page.parent, devel[0])
-
-
-class TestSnappyCurrentImport(TestLocalBranchImport):
-    def runTest(self):
-        self.create_repo('data/snappy-test')
-        snappy_page = db_add_empty_page('Snappy', self.root)
-        guides = db_add_empty_page('Guides', snappy_page)
-        publish_pages([snappy_page, guides])
-        self.assertTrue(isinstance(self.repo, SnappyRepo))
-        self.repo.add_directive('docs', 'snappy/guides/current')
-        self.assertTrue(self.repo.execute_import_directives())
-        self.assertTrue(self.repo.publish())
-        number_of_articles = len(self.repo.imported_articles)
-        for article in self.repo.imported_articles:
-            self.assertTrue(isinstance(article, SnappyArticle))
-        self.assertGreater(number_of_articles, 0)
-        pages = Page.objects.filter(publisher_is_draft=False)
-        current_search = [
-            a for a in pages
-            if a.get_slug('current') and
-            a.get_absolute_url().endswith('snappy/guides/current/')]
-        self.assertEqual(len(current_search), 1)
-        current = current_search[0]
-        nav_pages = [self.root, snappy_page, guides, current]
-        self.assertEqual(
-            number_of_articles, pages.count()-len(nav_pages))
-        for page in [a for a in pages if a not in nav_pages]:
-            if page.get_redirect(DEFAULT_LANG):
-                self.assertEqual(page.parent, guides)
-            else:
-                self.assertEqual(page.parent, current)

=== modified file 'md_importer/tests/test_utils.py'
--- md_importer/tests/test_utils.py	2016-01-11 14:02:04 +0000
+++ md_importer/tests/test_utils.py	2016-04-18 15:28:31 +0000
@@ -8,6 +8,7 @@
     db_add_empty_page,
     db_create_root_page,
     db_empty_page_list,
+    PublishedPages,
     TestLocalBranchImport,
 )
 
@@ -22,8 +23,8 @@
         root = db_create_root_page()
         self.assertNotEqual(root, None)
         self.assertFalse(root.publisher_is_draft)
-        self.assertEqual(
-            Page.objects.filter(publisher_is_draft=False).count(), 1)
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(1))
 
     def test_simple_articletree(self):
         db_empty_page_list()
@@ -33,8 +34,8 @@
         self.assertFalse(snappy.publisher_is_draft)
         guides = db_add_empty_page('Guides', snappy)
         self.assertFalse(guides.publisher_is_draft)
-        self.assertEqual(
-            Page.objects.filter(publisher_is_draft=False).count(), 3)
+        published_pages = PublishedPages()
+        self.assertTrue(published_pages.has_size(3))
         self.assertEqual(guides.parent.get_public_object(), snappy)
         self.assertEqual(snappy.parent.get_public_object(), root)
 

=== modified file 'md_importer/tests/test_website_import.py'
--- md_importer/tests/test_website_import.py	2016-04-04 09:04:22 +0000
+++ md_importer/tests/test_website_import.py	2016-04-18 15:28:31 +0000
@@ -2,11 +2,11 @@
 from md_importer.importer.repo import Repo
 from .utils import (
     db_add_empty_page,
+    PublishedPages,
     TestLocalBranchImport,
 )
 
 from cms.api import add_plugin, publish_pages
-from cms.models import Page
 
 
 class TestSnappyWebsiteRead(TestLocalBranchImport):
@@ -44,7 +44,7 @@
         self.assertTrue(self.repo.execute_import_directives())
         self.assertTrue(self.repo.publish())
         self.assertGreater(len(self.repo.pages), 10)
-        pages = Page.objects.filter(publisher_is_draft=False)
+        published_pages = PublishedPages()
         expected_urls = [
             '/en/',
             '/en/snappy/',
@@ -85,6 +85,6 @@
             '/en/snappy/start/as-dev/16-04/step4-first-snap/',
             '/en/snappy/start/as-dev/16-04/step5-further-readings/',
         ]
-        self.assertEqual(len(expected_urls), len(pages))
+        self.assertTrue(published_pages.has_size(len(expected_urls)))
         for url in expected_urls:
-            self.assertTrue(url in [p.get_absolute_url() for p in pages])
+            self.assertTrue(published_pages.contains_url(url))

=== modified file 'md_importer/tests/utils.py'
--- md_importer/tests/utils.py	2016-03-09 15:42:41 +0000
+++ md_importer/tests/utils.py	2016-04-18 15:28:31 +0000
@@ -10,8 +10,11 @@
 from cms.test_utils.testcases import CMSTestCase
 from cms.utils.page_resolver import get_page_from_request
 
-from ..importer import DEFAULT_LANG
-from ..importer.repo import create_repo
+from md_importer.models import ImportedArticle
+from ..importer import (
+    DEFAULT_LANG,
+)
+from ..importer.repo import Repo
 
 if sys.version_info.major == 2:
     from urlparse import urlparse
@@ -52,7 +55,7 @@
     def create_repo(self, docs_path):
         origin = os.path.join(os.path.dirname(__file__), docs_path)
         self.assertTrue(os.path.exists(origin))
-        self.repo = create_repo(self.tempdir, origin, '', '')
+        self.repo = Repo(self.tempdir, origin, '', '')
         self.fetch_retcode = self.repo.get()
         self.assertEqual(self.fetch_retcode, 0)
 
@@ -69,6 +72,18 @@
         shutil.rmtree(self.tempdir)
 
 
+def check_imported_article(imported_article):
+    assert imported_article is not None
+    assert isinstance(imported_article, ImportedArticle)
+    return True
+
+
+def check_repo(repo):
+    assert repo is not None
+    assert isinstance(repo, Repo)
+    return repo.assert_is_published()
+
+
 def is_local_link(link):
     if link.has_attr('class') and \
        'headeranchor-link' in link.attrs['class']:
@@ -78,3 +93,22 @@
     if scheme in ['http', 'https', 'mailto']:
         return False
     return True
+
+
+class PublishedPages:
+    def __init__(self):
+        self.pages = None
+        self.update()
+
+    def update(self):
+        self.pages = Page.objects.filter(publisher_is_draft=False)
+        self.urls = [p.get_absolute_url() for p in self.pages]
+
+    def contains_url(self, url):
+        return url in self.urls
+
+    def has_size(self, size):
+        return self.pages.count() == size
+
+    def has_at_least_size(self, size):
+        return self.pages.count() >= size