Merge lp:~cjwatson/launchpad/distribution-filebug-dsp-vocab into lp:launchpad

[Colin Watson]

Convert Distribution:+filebug and friends to use the DistributionSourcePackage picker if the appropriate feature flag is set.

Convert Distribution:+filebug and friends to use the DistributionSourcePackage picker if the appropriate feature flag is set.

The hardest bit of this was revealed by trying to extend an existing doctest to test this case: there are not entirely unreasonable use cases for adding bug tasks to existing bugs for packages in distributions whose set of packages we don't track at all, for example "this bug also affects the 'unzip' package in Gentoo and here's their bug on the subject". I think it's overkill to forbid this altogether, but we can and should be strict about distributions whose packages we track properly, and we should still not allow searching for SPNs even via other distributions. The compromise I found was to allow DistributionSourcePackageVocabulary.toTerm to return exact matches provided that the distribution has no rows at all in DistributionSourcePackageCache; this way it behaves as we want for at least Ubuntu, Debian, and charms, and e.g. Gentoo or openSUSE will get the more liberal treatment.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'database/sampledata/current-dev.sql'
--- database/sampledata/current-dev.sql	2016-07-22 11:23:31 +0000
+++ database/sampledata/current-dev.sql	2016-09-19 12:53:26 +0000
@@ -3500,7 +3500,7 @@
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (6, 1, 19, 'alsa-utils', '', '', '', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (7, 1, 20, 'cnews', '', '', '', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (8, 1, 21, 'libstdc++', '', '', '', NULL, NULL, 1);
-INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (9, 1, 22, 'linux-source-2.6.15', '', '', '', NULL, NULL, 1);
+INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (9, 1, 22, 'linux-source-2.6.15', 'linux-2.6.12', 'the kernel of boom', 'this kernel is like the crystal method: a temple of boom', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (10, 1, 23, 'foobar', '', '', '', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (11, 1, 27, 'commercialpackage', '', '', '', NULL, NULL, 12);
 

=== modified file 'database/sampledata/current.sql'
--- database/sampledata/current.sql	2016-07-22 10:48:21 +0000
+++ database/sampledata/current.sql	2016-09-19 12:53:26 +0000
@@ -3434,7 +3434,7 @@
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (6, 1, 19, 'alsa-utils', '', '', '', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (7, 1, 20, 'cnews', '', '', '', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (8, 1, 21, 'libstdc++', '', '', '', NULL, NULL, 1);
-INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (9, 1, 22, 'linux-source-2.6.15', '', '', '', NULL, NULL, 1);
+INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (9, 1, 22, 'linux-source-2.6.15', 'linux-2.6.12', 'the kernel of boom', 'this kernel is like the crystal method: a temple of boom', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (10, 1, 23, 'foobar', '', '', '', NULL, NULL, 1);
 INSERT INTO distributionsourcepackagecache (id, distribution, sourcepackagename, name, binpkgnames, binpkgsummaries, binpkgdescriptions, fti, changelog, archive) VALUES (11, 1, 27, 'commercialpackage', '', '', '', NULL, NULL, 12);
 

=== modified file 'lib/lp/bugs/browser/bugtarget.py'
--- lib/lp/bugs/browser/bugtarget.py	2016-04-29 11:11:35 +0000
+++ lib/lp/bugs/browser/bugtarget.py	2016-09-19 12:53:26 +0000
@@ -91,6 +91,7 @@
     BugTagsWidget,
     LargeBugTagsWidget,
     )
+from lp.bugs.browser.widgets.bugtask import FileBugSourcePackageNameWidget
 from lp.bugs.interfaces.apportjob import IProcessApportBlobJobSource
 from lp.bugs.interfaces.bug import (
     CreateBugParams,
@@ -131,6 +132,7 @@
 from lp.registry.interfaces.sourcepackage import ISourcePackage
 from lp.registry.vocabularies import ValidPersonOrTeamVocabulary
 from lp.services.config import config
+from lp.services.features import getFeatureFlag
 from lp.services.job.interfaces.job import JobStatus
 from lp.services.librarian.browser import ProxiedLibraryFileAlias
 from lp.services.propertycache import cachedproperty
@@ -225,6 +227,7 @@
 
     custom_widget('information_type', LaunchpadRadioWidgetWithDescription)
     custom_widget('comment', TextAreaWidget, cssClass='comment-text')
+    custom_widget('packagename', FileBugSourcePackageNameWidget)
 
     extra_data_token = None
 
@@ -419,8 +422,17 @@
                     distribution = self.context.distribution
 
                 try:
-                    distribution.guessPublishedSourcePackageName(packagename)
-                except NotFoundError:
+                    if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+                        dsp_vocab = self.widgets.get("packagename").vocabulary
+                        dsp_vocab.setDistribution(distribution)
+                        dsp_vocab.getTermByToken(packagename)
+                    else:
+                        # The untrusted BinaryAndSourcePackageName
+                        # vocabulary was used, so it needs secondary
+                        # verification.
+                        distribution.guessPublishedSourcePackageName(
+                            packagename)
+                except (LookupError, NotFoundError):
                     if distribution.series:
                         # If a distribution doesn't have any series,
                         # it won't have any source packages published at
@@ -525,24 +537,28 @@
             information_type=information_type,
             tags=data.get('tags'))
         if IDistribution.providedBy(context) and packagename:
-            # We don't know if the package name we got was a source or binary
-            # package name, so let the Soyuz API figure it out for us.
-            packagename = str(packagename.name)
-            try:
-                sourcepackagename = context.guessPublishedSourcePackageName(
-                    packagename)
-            except NotFoundError:
-                notifications.append(
-                    "The package %s is not published in %s; the "
-                    "bug was targeted only to the distribution."
-                    % (packagename, context.displayname))
-                params.comment += (
-                    "\r\n\r\nNote: the original reporter indicated "
-                    "the bug was in package %r; however, that package "
-                    "was not published in %s." % (
-                        packagename, context.displayname))
+            if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+                context = packagename
             else:
-                context = context.getSourcePackage(sourcepackagename.name)
+                # We don't know if the package name we got was a source or
+                # binary package name, so let the Soyuz API figure it out
+                # for us.
+                packagename = str(packagename.name)
+                try:
+                    sourcepackagename = (
+                        context.guessPublishedSourcePackageName(packagename))
+                except NotFoundError:
+                    notifications.append(
+                        "The package %s is not published in %s; the "
+                        "bug was targeted only to the distribution."
+                        % (packagename, context.displayname))
+                    params.comment += (
+                        "\r\n\r\nNote: the original reporter indicated "
+                        "the bug was in package %r; however, that package "
+                        "was not published in %s." % (
+                            packagename, context.displayname))
+                else:
+                    context = context.getSourcePackage(sourcepackagename.name)
 
         extra_data = self.extra_data
         if extra_data.extra_description:
@@ -895,13 +911,25 @@
             filebug_url, status=httplib.MOVED_PERMANENTLY)
 
 
+class IDistroBugAddForm(IBugAddForm):
+
+    packagename = copy_field(
+        IBugAddForm['packagename'], vocabularyName='DistributionSourcePackage')
+
+
 class FilebugShowSimilarBugsView(FileBugViewBase):
     """A view for showing possible dupes for a bug.
 
     This view will only be used to populate asynchronously-driven parts
     of a page.
     """
-    schema = IBugAddForm
+
+    @property
+    def schema(self):
+        if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+            return IDistroBugAddForm
+        else:
+            return IBugAddForm
 
     # XXX: Brad Bollenbach 2006-10-04: This assignment to actions is a
     # hack to make the action decorator Just Work across inheritance.

=== modified file 'lib/lp/bugs/browser/tests/test_bugtarget_filebug.py'
--- lib/lp/bugs/browser/tests/test_bugtarget_filebug.py	2016-01-26 15:47:37 +0000
+++ lib/lp/bugs/browser/tests/test_bugtarget_filebug.py	2016-09-19 12:53:26 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010-2012 Canonical Ltd.  This software is licensed under the
+# Copyright 2010-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
@@ -7,6 +7,10 @@
 
 from BeautifulSoup import BeautifulSoup
 from lazr.restful.interfaces import IJSONRequestCache
+from testscenarios import (
+    load_tests_apply_scenarios,
+    WithScenarios,
+    )
 import transaction
 from zope.component import getUtility
 from zope.publisher.interfaces import NotFound
@@ -32,6 +36,7 @@
     )
 from lp.registry.enums import BugSharingPolicy
 from lp.registry.interfaces.projectgroup import IProjectGroup
+from lp.services.features.testing import FeatureFixture
 from lp.services.temporaryblobstorage.interfaces import (
     ITemporaryStorageManager,
     )
@@ -787,10 +792,22 @@
             soup.find('input', attrs={'name': 'field.information_type'}))
 
 
-class TestFileBugSourcePackage(TestCaseWithFactory):
+class TestFileBugSourcePackage(WithScenarios, TestCaseWithFactory):
 
     layer = DatabaseFunctionalLayer
 
+    scenarios = [
+        ("bspn_picker", {"features": {}}),
+        ("dsp_picker", {
+            "features": {u"disclosure.dsp_picker.enabled": u"on"},
+            }),
+        ]
+
+    def setUp(self):
+        super(TestFileBugSourcePackage, self).setUp()
+        if self.features:
+            self.useFixture(FeatureFixture(self.features))
+
     def test_filebug_works_on_official_package_branch(self):
         # It should be possible to file a bug against a source package
         # when there is an official package branch.
@@ -936,3 +953,6 @@
         login_person(user)
         view = create_initialized_view(product, '+filebug', principal=user)
         self._assert_cache_values(view, False)
+
+
+load_tests = load_tests_apply_scenarios

=== modified file 'lib/lp/bugs/browser/widgets/bugtask.py'
--- lib/lp/bugs/browser/widgets/bugtask.py	2016-07-23 10:28:41 +0000
+++ lib/lp/bugs/browser/widgets/bugtask.py	2016-09-19 12:53:26 +0000
@@ -13,6 +13,7 @@
     "BugTaskTargetWidget",
     "BugWatchEditForm",
     "DBItemDisplayWidget",
+    "FileBugSourcePackageNameWidget",
     "NewLineToSpacesWidget",
     "UbuntuSourcePackageNameWidget",
     ]
@@ -42,6 +43,7 @@
     InvalidValue,
     ValidationError,
     )
+from zope.schema.vocabulary import getVocabularyRegistry
 
 from lp import _
 from lp.app.browser.tales import TeamFormatterAPI
@@ -66,7 +68,10 @@
     UnrecognizedBugTrackerURL,
     )
 from lp.bugs.vocabularies import UsesBugsDistributionVocabulary
-from lp.registry.interfaces.distribution import IDistributionSet
+from lp.registry.interfaces.distribution import (
+    IDistribution,
+    IDistributionSet,
+    )
 from lp.services.features import getFeatureFlag
 from lp.services.fields import URIField
 from lp.services.webapp import canonical_url
@@ -497,7 +502,7 @@
     def getDistribution(self):
         """Get the distribution used for package validation.
 
-        The package name has be to published in the returned distribution.
+        The package name has to be published in the returned distribution.
         """
         field = self.context
         distribution = field.context.distribution
@@ -543,14 +548,14 @@
     BugTaskSourcePackageNameWidget):
     """Package widget for +distrotask.
 
-    This widgets works the same as `BugTaskSourcePackageNameWidget`,
-    except that it gets the distribution from the request.
+    This widget works the same as `BugTaskSourcePackageNameWidget`, except
+    that it gets the distribution from the request.
     """
 
     distribution_id = 'field.distribution'
 
     def getDistribution(self):
-        """See `BugTaskSourcePackageNameWidget`"""
+        """See `BugTaskSourcePackageNameWidget`."""
         distribution_name = self.request.form.get('field.distribution')
         if distribution_name is None:
             raise UnexpectedFormData(
@@ -563,6 +568,42 @@
         return distribution
 
 
+class FileBugSourcePackageNameWidget(BugTaskSourcePackageNameWidget):
+    """Package widget for +filebug.
+
+    This widget works the same as `BugTaskSourcePackageNameWidget`, except
+    that it expects the field's context to be a bug target rather than a bug
+    task.
+    """
+
+    def getDistribution(self):
+        """See `BugTaskSourcePackageNameWidget`."""
+        field = self.context
+        pillar = field.context.pillar
+        assert IDistribution.providedBy(pillar), (
+            "FileBugSourcePackageNameWidget should be used only for"
+            " distribution bug targets.")
+        return pillar
+
+    def _toFieldValue(self, input):
+        """See `BugTaskSourcePackageNameWidget`."""
+        source = super(FileBugSourcePackageNameWidget, self)._toFieldValue(
+            input)
+        if (source is not None and
+                not bool(getFeatureFlag('disclosure.dsp_picker.enabled'))):
+            # XXX cjwatson 2016-07-25: Convert to a value that the
+            # IBug.packagename vocabulary will accept.  This is a fiddly
+            # hack, but it only needs to survive until we can switch to the
+            # DistributionSourcePackage picker across the board.
+            bspn_vocab = getVocabularyRegistry().get(
+                None, "BinaryAndSourcePackageName")
+            bspn = bspn_vocab.getTermByToken(source.name).value
+            self.cached_values[input] = bspn
+            return bspn
+        else:
+            return source
+
+
 class UbuntuSourcePackageNameWidget(BugTaskSourcePackageNameWidget):
     """A widget to select Ubuntu packages."""
 

=== modified file 'lib/lp/bugs/doc/bugtask-package-widget.txt'
--- lib/lp/bugs/doc/bugtask-package-widget.txt	2011-12-24 17:49:30 +0000
+++ lib/lp/bugs/doc/bugtask-package-widget.txt	2016-09-19 12:53:26 +0000
@@ -8,13 +8,17 @@
 package name, and to convert it to a source package name, we have a
 custom widget.
 
+    >>> from lazr.restful.interface import copy_field
     >>> from lp.bugs.browser.widgets.bugtask import (
     ...     BugTaskSourcePackageNameWidget)
+    >>> from lp.registry.interfaces.distribution import IDistributionSet
+    >>> from lp.services.features import getFeatureFlag
+    >>> from lp.testing import person_logged_in
 
 If we pass a valid source package name to it, the corresponding
-SourcePackageName will be returned by getInputValue(). In order for us
-to map the package names, we need a distribution, so we give the widget
-a distribution task to work with.
+SourcePackageName (or DistributionSourcePackage, for the new picker) will be
+returned by getInputValue(). In order for us to map the package names, we
+need a distribution, so we give the widget a distribution task to work with.
 
     >>> from lp.services.webapp.servers import LaunchpadTestRequest
     >>> from lp.bugs.interfaces.bug import IBugSet
@@ -24,52 +28,67 @@
     >>> ubuntu_task.distribution.name
     u'ubuntu'
 
-    >>> package_field = IBugTask['sourcepackagename'].bind(ubuntu_task)
+    >>> unbound_package_field = IBugTask['sourcepackagename']
+    >>> if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+    ...     unbound_package_field = copy_field(
+    ...         unbound_package_field,
+    ...         vocabularyName='DistributionSourcePackage')
+    ...     expected_input_class = 'DistributionSourcePackage'
+    ... else:
+    ...     expected_input_class = 'SourcePackageName'
+    >>> package_field = unbound_package_field.bind(ubuntu_task)
 
     >>> request = LaunchpadTestRequest(
     ...     form={'field.sourcepackagename': 'evolution'})
     >>> widget = BugTaskSourcePackageNameWidget(
     ...     package_field, package_field.vocabulary, request)
-    >>> widget.getInputValue()
-    <SourcePackageName ...>
+    >>> widget.getInputValue().__class__.__name__ == expected_input_class
+    True
     >>> widget.getInputValue().name
     u'evolution'
 
-
-If we pass in a binary package name, which can be mapped to a source
-package name, the corresponding SourcePackageName is returned.
-
+If we pass in a binary package name, which can be mapped to a source package
+name, the corresponding SourcePackageName is returned.  (In the case of the
+new picker, this instead requires searching first.)
+
+    >>> package_name = 'linux-2.6.12'
+    >>> if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+    ...     package_field.vocabulary.setDistribution(ubuntu_task.distribution)
+    ...     results = package_field.vocabulary.searchForTerms(package_name)
+    ...     package_name = list(results)[0].value
     >>> request = LaunchpadTestRequest(
-    ...     form={'field.sourcepackagename': 'linux-2.6.12'})
+    ...     form={'field.sourcepackagename': package_name})
     >>> widget = BugTaskSourcePackageNameWidget(
     ...     package_field, package_field.vocabulary, request)
-    >>> widget.getInputValue()
-    <SourcePackageName ...>
+    >>> widget.getInputValue().__class__.__name__ == expected_input_class
+    True
     >>> widget.getInputValue().name
     u'linux-source-2.6.15'
 
-For some distribution we don't know exactly which source packages it
-contains, so IDistribution.guessPublishedSourcePackageName will raise a
+For some distributions we don't know exactly which source packages they
+contain, so IDistribution.guessPublishedSourcePackageName will raise a
 NotFoundError.
 
-    >>> debian_task = bug_one.bugtasks[-1]
-    >>> debian_task.distribution.name
-    u'debian'
-    >>> debian_task.distribution.guessPublishedSourcePackageName('evolution')
+    >>> gentoo = getUtility(IDistributionSet)['gentoo']
+    >>> gentoo.guessPublishedSourcePackageName('evolution')
     Traceback (most recent call last):
     ...
     NotFoundError...
 
-At that point we'll fallback to the vocabulary, so a SourcePackageName
+At that point we'll fall back to the vocabulary, so a SourcePackageName
 will still be returned.
 
-    >>> package_field = IBugTask['sourcepackagename'].bind(debian_task)
+    >>> with person_logged_in(ubuntu_task.owner):
+    ...     gentoo_task = bug_one.addTask(ubuntu_task.owner, gentoo)
+    >>> package_field = unbound_package_field.bind(gentoo_task)
+    >>> if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+    ...     package_field.vocabulary.setDistribution(gentoo)
     >>> request = LaunchpadTestRequest(
     ...     form={'field.sourcepackagename': 'evolution'})
     >>> widget = BugTaskSourcePackageNameWidget(
     ...     package_field, package_field.vocabulary, request)
-    >>> widget.getInputValue()
-    <SourcePackageName ...>
+    >>> widget.getInputValue().__class__.__name__ == expected_input_class
+    True
     >>> widget.getInputValue().name
     u'evolution'
 
@@ -126,3 +145,89 @@
     Traceback (most recent call last):
     ...
     UnexpectedFormData: ...
+
+
+FileBugSourcePackageNameWidget
+------------------------------
+
+The +filebug page uses a widget that works much the same way as
+BugTaskSourcePackageNameWidget, except that in this case the context is a
+bug target rather than a bug task.
+
+    >>> from lp.bugs.browser.widgets.bugtask import (
+    ...     FileBugSourcePackageNameWidget)
+    >>> from lp.bugs.interfaces.bug import IBugAddForm
+
+    >>> unbound_package_field = IBugAddForm['packagename']
+    >>> if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+    ...     unbound_package_field = copy_field(
+    ...         unbound_package_field,
+    ...         vocabularyName='DistributionSourcePackage')
+    ...     expected_input_class = 'DistributionSourcePackage'
+    ... else:
+    ...     expected_input_class = 'BinaryAndSourcePackageName'
+    >>> package_field = unbound_package_field.bind(ubuntu_task.distribution)
+
+    >>> request = LaunchpadTestRequest(
+    ...     form={'field.packagename': 'evolution'})
+    >>> widget = FileBugSourcePackageNameWidget(
+    ...     package_field, package_field.vocabulary, request)
+    >>> widget.getInputValue().__class__.__name__ == expected_input_class
+    True
+    >>> widget.getInputValue().name
+    u'evolution'
+
+If we pass in a binary package name, which can be mapped to a source
+package name, the corresponding source package name (albeit as a
+BinaryAndSourcePackageName) is returned.  (In the case of the new picker,
+this instead requires searching first.)
+
+    >>> package_name = 'linux-2.6.12'
+    >>> if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+    ...     package_field.vocabulary.setDistribution(ubuntu_task.distribution)
+    ...     results = package_field.vocabulary.searchForTerms(package_name)
+    ...     package_name = list(results)[0].value
+    >>> request = LaunchpadTestRequest(
+    ...     form={'field.packagename': package_name})
+    >>> widget = FileBugSourcePackageNameWidget(
+    ...     package_field, package_field.vocabulary, request)
+    >>> widget.getInputValue().__class__.__name__ == expected_input_class
+    True
+    >>> widget.getInputValue().name
+    u'linux-source-2.6.15'
+
+For some distributions we don't know exactly which source packages they
+contain, so IDistribution.guessPublishedSourcePackageName will raise a
+NotFoundError.
+
+    >>> gentoo_task.distribution.guessPublishedSourcePackageName('evolution')
+    Traceback (most recent call last):
+    ...
+    NotFoundError...
+
+At that point we'll fall back to the vocabulary, so a SourcePackageName
+will still be returned.
+
+    >>> package_field = unbound_package_field.bind(gentoo_task.distribution)
+    >>> if bool(getFeatureFlag('disclosure.dsp_picker.enabled')):
+    ...     package_field.vocabulary.setDistribution(gentoo)
+    >>> request = LaunchpadTestRequest(
+    ...     form={'field.packagename': 'evolution'})
+    >>> widget = FileBugSourcePackageNameWidget(
+    ...     package_field, package_field.vocabulary, request)
+    >>> widget.getInputValue().__class__.__name__ == expected_input_class
+    True
+    >>> widget.getInputValue().name
+    u'evolution'
+
+If we pass in a package name that doesn't exist in Launchpad, we get a
+ConversionError saying that the package name doesn't exist.
+
+    >>> request = LaunchpadTestRequest(
+    ...     form={'field.packagename': 'no-package'})
+    >>> widget = FileBugSourcePackageNameWidget(
+    ...     package_field, package_field.vocabulary, request)
+    >>> widget.getInputValue()
+    Traceback (most recent call last):
+    ...
+    ConversionError...

=== modified file 'lib/lp/bugs/tests/test_doc.py'
--- lib/lp/bugs/tests/test_doc.py	2012-10-08 06:13:17 +0000
+++ lib/lp/bugs/tests/test_doc.py	2016-09-19 12:53:26 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2012 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """
@@ -11,6 +11,7 @@
 
 from lp.code.tests.test_doc import branchscannerSetUp
 from lp.services.config import config
+from lp.services.features.testing import FeatureFixture
 from lp.services.mail.tests.test_doc import ProcessMailLayer
 from lp.soyuz.tests.test_doc import (
     lobotomize_stevea,
@@ -128,6 +129,18 @@
     login('[e-mail]')
 
 
+def enableDSPPickerSetUp(test):
+    setUp(test)
+    ff = FeatureFixture({u'disclosure.dsp_picker.enabled': u'on'})
+    ff.setUp()
+    test.globs['dsp_picker_feature_fixture'] = ff
+
+
+def enableDSPPickerTearDown(test):
+    test.globs['dsp_picker_feature_fixture'].cleanUp()
+    tearDown(test)
+
+
 special = {
     'cve-update.txt': LayeredDocFileSuite(
         '../doc/cve-update.txt',
@@ -206,6 +219,18 @@
         tearDown=tearDown,
         layer=LaunchpadZopelessLayer
         ),
+    'bugtask-package-widget.txt': LayeredDocFileSuite(
+        '../doc/bugtask-package-widget.txt',
+        id_extensions=['bugtask-package-widget.txt'],
+        setUp=setUp, tearDown=tearDown,
+        layer=LaunchpadFunctionalLayer
+        ),
+    'bugtask-package-widget.txt-dsp-picker': LayeredDocFileSuite(
+        '../doc/bugtask-package-widget.txt',
+        id_extensions=['bugtask-package-widget.txt-dsp-picker'],
+        setUp=enableDSPPickerSetUp, tearDown=enableDSPPickerTearDown,
+        layer=LaunchpadFunctionalLayer
+        ),
     'bugmessage.txt': LayeredDocFileSuite(
         '../doc/bugmessage.txt',
         id_extensions=['bugmessage.txt'],

=== modified file 'lib/lp/registry/tests/test_distributionsourcepackage_vocabulary.py'
--- lib/lp/registry/tests/test_distributionsourcepackage_vocabulary.py	2016-09-13 12:48:02 +0000
+++ lib/lp/registry/tests/test_distributionsourcepackage_vocabulary.py	2016-09-19 12:53:26 +0000
@@ -86,18 +86,32 @@
         vocabulary = DistributionSourcePackageVocabulary(dsp)
         self.assertIn(dsp, vocabulary)
 
+    def test_contains_true_with_cacheless_distribution(self):
+        # The vocabulary contains DSPs that are not official, provided that
+        # the distribution has no cached package names.
+        dsp = self.factory.makeDistributionSourcePackage(with_db=False)
+        vocabulary = DistributionSourcePackageVocabulary(dsp.distribution)
+        self.assertIn(dsp, vocabulary)
+
     def test_contains_false_with_distribution(self):
         # The vocabulary does not contain DSPs that are not official that
         # were not passed to init.
-        dsp = self.factory.makeDistributionSourcePackage(with_db=False)
+        distro = self.factory.makeDistribution()
+        distroseries = self.factory.makeDistroSeries(distribution=distro)
+        self.factory.makeDSPCache(distroseries=distroseries)
+        dsp = self.factory.makeDistributionSourcePackage(
+            distribution=distro, with_db=False)
         vocabulary = DistributionSourcePackageVocabulary(dsp.distribution)
         self.assertNotIn(dsp, vocabulary)
 
     def test_toTerm_raises_error(self):
         # An error is raised for DSP/SPNs that are not official and are not
         # in the vocabulary.
+        distro = self.factory.makeDistribution()
+        distroseries = self.factory.makeDistroSeries(distribution=distro)
+        self.factory.makeDSPCache(distroseries=distroseries)
         dsp = self.factory.makeDistributionSourcePackage(
-            sourcepackagename='foo')
+            sourcepackagename='foo', distribution=distro, with_db=False)
         vocabulary = DistributionSourcePackageVocabulary(dsp.distribution)
         self.assertRaises(LookupError, vocabulary.toTerm, dsp)
 
@@ -118,6 +132,18 @@
         self.assertEqual(dsp.name, term.title)
         self.assertEqual(dsp, term.value)
 
+    def test_toTerm_spn_with_cacheless_distribution(self):
+        # An SPN with no official DSP is accepted, provided that the
+        # distribution has no cached package names.
+        distro = self.factory.makeDistribution()
+        spn = self.factory.makeSourcePackageName()
+        vocabulary = DistributionSourcePackageVocabulary(distro)
+        term = vocabulary.toTerm(spn)
+        self.assertEqual(spn.name, term.token)
+        self.assertEqual(spn.name, term.title)
+        self.assertEqual(distro, term.value.distribution)
+        self.assertEqual(spn, term.value.sourcepackagename)
+
     def test_toTerm_dsp(self):
         # The DSP's distribution is used when a DSP is passed.
         spph = self.factory.makeSourcePackagePublishingHistory()
@@ -142,10 +168,24 @@
         self.assertEqual(dsp, term.value)
         self.assertEqual(['one', 'two'], term.value.binary_names)
 
+    def test_toTerm_dsp_with_cacheless_distribution(self):
+        # A DSP that is not official is accepted, provided that the
+        # distribution has no cached package names.
+        dsp = self.factory.makeDistributionSourcePackage(
+            sourcepackagename='foo', with_db=False)
+        vocabulary = DistributionSourcePackageVocabulary(dsp.distribution)
+        term = vocabulary.toTerm(dsp)
+        self.assertEqual(dsp.name, term.token)
+        self.assertEqual(dsp.name, term.title)
+        self.assertEqual(dsp, term.value)
+
     def test_getTermByToken_error(self):
         # An error is raised if the token does not match a official DSP.
+        distro = self.factory.makeDistribution()
+        distroseries = self.factory.makeDistroSeries(distribution=distro)
+        self.factory.makeDSPCache(distroseries=distroseries)
         dsp = self.factory.makeDistributionSourcePackage(
-            sourcepackagename='foo')
+            distribution=distro, sourcepackagename='foo', with_db=False)
         vocabulary = DistributionSourcePackageVocabulary(dsp.distribution)
         self.assertRaises(LookupError, vocabulary.getTermByToken, dsp.name)
 
@@ -158,6 +198,15 @@
         term = vocabulary.getTermByToken(dsp.name)
         self.assertEqual(dsp, term.value)
 
+    def test_getTermByToken_token_with_cacheless_distribution(self):
+        # The term is returned if it does not match an official DSP,
+        # provided that the distribution has no cached package names.
+        dsp = self.factory.makeDistributionSourcePackage(
+            sourcepackagename='foo', with_db=False)
+        vocabulary = DistributionSourcePackageVocabulary(dsp.distribution)
+        term = vocabulary.getTermByToken(dsp.name)
+        self.assertEqual(dsp, term.value)
+
     def test_searchForTerms_without_distribution(self):
         # searchForTerms asserts that the vocabulary has a distribution.
         spph = self.factory.makeSourcePackagePublishingHistory()

=== modified file 'lib/lp/registry/vocabularies.py'
--- lib/lp/registry/vocabularies.py	2016-09-13 12:48:02 +0000
+++ lib/lp/registry/vocabularies.py	2016-09-19 12:53:26 +0000
@@ -2071,6 +2071,16 @@
                 "DistributionSourcePackageVocabulary cannot be used without "
                 "setting a distribution.")
 
+    @property
+    def _cache_location_clauses(self):
+        return [
+            Or(
+                DistributionSourcePackageCache.archiveID.is_in(
+                    self.distribution.all_distro_archive_ids),
+                DistributionSourcePackageCache.archive == None),
+            DistributionSourcePackageCache.distribution == self.distribution,
+            ]
+
     def toTerm(self, spn_or_dsp):
         """See `IVocabulary`."""
         self._assertHasDistribution()
@@ -2089,18 +2099,34 @@
             dsp = spn_or_dsp
         elif spn_or_dsp is not None:
             dsp = self.distribution.getSourcePackage(spn_or_dsp)
-        if dsp is not None and (dsp == self.dsp or dsp.is_official):
-            if binary_names:
-                # Search already did the hard work of looking up binary names.
-                cache = get_property_cache(dsp)
-                cache.binary_names = binary_names
-            # XXX cjwatson 2016-07-22: It's a bit odd for the token to
-            # return just the source package name and not the distribution
-            # name as well, but at the moment this is always fed into a
-            # package name box so things work much better this way.  If we
-            # ever do a true combined distribution/package picker, then this
-            # may need to be revisited.
-            return SimpleTerm(dsp, dsp.name, dsp.name)
+        if dsp is not None:
+            if dsp == self.dsp or dsp.is_official:
+                if binary_names:
+                    # Search already did the hard work of looking up binary
+                    # names.
+                    cache = get_property_cache(dsp)
+                    cache.binary_names = binary_names
+                # XXX cjwatson 2016-07-22: It's a bit odd for the token to
+                # return just the source package name and not the
+                # distribution name as well, but at the moment this is
+                # always fed into a package name box so things work much
+                # better this way.  If we ever do a true combined
+                # distribution/package picker, then this may need to be
+                # revisited.
+                return SimpleTerm(dsp, dsp.name, dsp.name)
+            else:
+                # Does this vocabulary have any package names at all?
+                empty = IStore(DistributionSourcePackageCache).find(
+                    DistributionSourcePackageCache.sourcepackagenameID,
+                    *self._cache_location_clauses).is_empty()
+                if empty:
+                    # If the vocabulary has no package names, then this is
+                    # probably a distribution not managed in Launchpad.  In
+                    # that case we are more liberal about allowing unknown
+                    # package names, in order to support existing uses such
+                    # as noting that the same bug exists in the same package
+                    # in multiple distributions.
+                    return SimpleTerm(dsp, dsp.name, dsp.name)
         raise LookupError(self.distribution, spn_or_dsp)
 
     def getTerm(self, spn_or_dsp):
@@ -2134,13 +2160,7 @@
                     DistributionSourcePackageCache.binpkgnames.contains_string(
                         query),
                     ),
-                Or(
-                    DistributionSourcePackageCache.archiveID.is_in(
-                        self.distribution.all_distro_archive_ids),
-                    DistributionSourcePackageCache.archive == None),
-                DistributionSourcePackageCache.distribution ==
-                    self.distribution,
-                ),
+                *self._cache_location_clauses),
             tables=DistributionSourcePackageCache,
             distinct=(DistributionSourcePackageCache.name,)))
         SearchableDSPC = Table("SearchableDSPC")