Merge lp:~cjwatson/launchpad/github-link into lp:launchpad

Proposed by Colin Watson on 2016-07-04
Status: Merged
Merged at revision: 18130
Proposed branch: lp:~cjwatson/launchpad/github-link
Merge into: lp:launchpad
Diff against target: 1401 lines (+876/-220)
15 files modified
lib/lp/bugs/configure.zcml (+12/-1)
lib/lp/bugs/doc/externalbugtracker-bugzilla-lp-plugin.txt (+1/-2)
lib/lp/bugs/doc/externalbugtracker-bugzilla.txt (+1/-1)
lib/lp/bugs/doc/externalbugtracker-trac-lp-plugin.txt (+2/-4)
lib/lp/bugs/doc/externalbugtracker-trac.txt (+1/-2)
lib/lp/bugs/doc/externalbugtracker.txt (+6/-5)
lib/lp/bugs/externalbugtracker/__init__.py (+33/-9)
lib/lp/bugs/externalbugtracker/base.py (+3/-2)
lib/lp/bugs/externalbugtracker/github.py (+264/-0)
lib/lp/bugs/externalbugtracker/tests/test_github.py (+375/-0)
lib/lp/bugs/interfaces/bugtracker.py (+8/-1)
lib/lp/bugs/model/bugtracker.py (+6/-1)
lib/lp/bugs/model/bugwatch.py (+15/-1)
lib/lp/bugs/stories/bugtracker/xx-bugtracker.txt (+1/-0)
lib/lp/bugs/tests/test_bugwatch.py (+148/-191)
To merge this branch: bzr merge lp:~cjwatson/launchpad/github-link
Reviewer Review Type Date Requested Status
Thomi Richards (community) Approve on 2016-07-11
Launchpad code reviewers 2016-07-04 Pending
Review via email: mp+299074@code.launchpad.net

Commit message

Add basic GitHub bug linking.

Description of the change

Add basic GitHub bug linking.

Each repository with linked issues needs its own BugTracker row, as GitHub namespaces issue numbers per repository rather than globally. However, our rate limit for GitHub API requests is global, and GitHub's documentation indicates that they may take punitive measures against clients that don't do a good job of backing off when they exceed their rate limit. This necessitates a GitHubRateLimit utility to keep track of how much of our limit we have left.

Our model of remote bug state is very simple for now. We map "open" to NEW, "closed" to FIXRELEASED, and ignore importance altogether. However, we do at least record the remote bug's labels even though we don't currently map them to anything.

On deployment, we'll want to get sysadmins to create a bot account for us on GitHub and generate an OAuth token. While unauthenticated access will minimally work, GitHub sets a much lower rate limit for that than for authenticated access.

To post a comment you must log in.
Thomi Richards (thomir) wrote :

LGTM - This is another branch where we're inheriting test cases, which makes me sad...

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/bugs/configure.zcml'
2--- lib/lp/bugs/configure.zcml 2016-02-04 12:45:38 +0000
3+++ lib/lp/bugs/configure.zcml 2016-07-13 10:12:00 +0000
4@@ -1,4 +1,4 @@
5-<!-- Copyright 2009-2012 Canonical Ltd. This software is licensed under the
6+<!-- Copyright 2009-2016 Canonical Ltd. This software is licensed under the
7 GNU Affero General Public License version 3 (see the file LICENSE).
8 -->
9
10@@ -518,6 +518,17 @@
11 interface="lp.bugs.interfaces.bugtracker.IRemoteBug"/>
12 </class>
13
14+ <!-- GitHubRateLimit -->
15+
16+ <class
17+ class="lp.bugs.externalbugtracker.github.GitHubRateLimit">
18+ <allow
19+ interface="lp.bugs.externalbugtracker.github.IGitHubRateLimit"/>
20+ </class>
21+ <utility
22+ factory="lp.bugs.externalbugtracker.github.GitHubRateLimit"
23+ provides="lp.bugs.externalbugtracker.github.IGitHubRateLimit"/>
24+
25 <!-- IBugBranch -->
26
27 <class
28
29=== modified file 'lib/lp/bugs/doc/externalbugtracker-bugzilla-lp-plugin.txt'
30--- lib/lp/bugs/doc/externalbugtracker-bugzilla-lp-plugin.txt 2012-12-26 01:32:19 +0000
31+++ lib/lp/bugs/doc/externalbugtracker-bugzilla-lp-plugin.txt 2016-07-13 10:12:00 +0000
32@@ -7,8 +7,7 @@
33 For testing purposes, a custom XML-RPC transport can be passed to it,
34 so that we can avoid network traffic in tests.
35
36- >>> from lp.bugs.externalbugtracker import (
37- ... BugzillaLPPlugin)
38+ >>> from lp.bugs.externalbugtracker.bugzilla import BugzillaLPPlugin
39 >>> from lp.bugs.tests.externalbugtracker import (
40 ... TestBugzillaXMLRPCTransport)
41 >>> test_transport = TestBugzillaXMLRPCTransport('http://example.com/')
42
43=== modified file 'lib/lp/bugs/doc/externalbugtracker-bugzilla.txt'
44--- lib/lp/bugs/doc/externalbugtracker-bugzilla.txt 2012-12-26 01:32:19 +0000
45+++ lib/lp/bugs/doc/externalbugtracker-bugzilla.txt 2016-07-13 10:12:00 +0000
46@@ -113,7 +113,7 @@
47
48 >>> transaction.commit()
49
50- >>> from lp.bugs.externalbugtracker import (
51+ >>> from lp.bugs.externalbugtracker.bugzilla import (
52 ... BugzillaAPI, BugzillaLPPlugin)
53 >>> bugzilla_to_use = bugzilla.getExternalBugTrackerToUse()
54
55
56=== modified file 'lib/lp/bugs/doc/externalbugtracker-trac-lp-plugin.txt'
57--- lib/lp/bugs/doc/externalbugtracker-trac-lp-plugin.txt 2012-11-02 03:23:34 +0000
58+++ lib/lp/bugs/doc/externalbugtracker-trac-lp-plugin.txt 2016-07-13 10:12:00 +0000
59@@ -7,10 +7,8 @@
60 For testing purposes, a custom XML-RPC transport can be passed to it,
61 so that we can avoid network traffic in tests.
62
63- >>> from lp.bugs.externalbugtracker import (
64- ... TracLPPlugin)
65- >>> from lp.bugs.tests.externalbugtracker import (
66- ... TestTracXMLRPCTransport)
67+ >>> from lp.bugs.externalbugtracker.trac import TracLPPlugin
68+ >>> from lp.bugs.tests.externalbugtracker import TestTracXMLRPCTransport
69 >>> test_transport = TestTracXMLRPCTransport('http://example.com/')
70 >>> trac = TracLPPlugin(
71 ... 'http://example.com/', xmlrpc_transport=test_transport)
72
73=== modified file 'lib/lp/bugs/doc/externalbugtracker-trac.txt'
74--- lib/lp/bugs/doc/externalbugtracker-trac.txt 2012-12-26 01:32:19 +0000
75+++ lib/lp/bugs/doc/externalbugtracker-trac.txt 2016-07-13 10:12:00 +0000
76@@ -41,8 +41,7 @@
77 >>> chosen_trac = trac.getExternalBugTrackerToUse()
78 http://example.com/launchpad-auth/check
79
80- >>> from lp.bugs.externalbugtracker import (
81- ... TracLPPlugin)
82+ >>> from lp.bugs.externalbugtracker.trac import TracLPPlugin
83 >>> isinstance(chosen_trac, TracLPPlugin)
84 True
85 >>> chosen_trac.baseurl
86
87=== modified file 'lib/lp/bugs/doc/externalbugtracker.txt'
88--- lib/lp/bugs/doc/externalbugtracker.txt 2016-02-05 16:51:12 +0000
89+++ lib/lp/bugs/doc/externalbugtracker.txt 2016-07-13 10:12:00 +0000
90@@ -49,8 +49,7 @@
91 instance. Usually there is only one version, so the default for the
92 original instance is to return itself.
93
94- >>> from lp.bugs.externalbugtracker import (
95- ... ExternalBugTracker)
96+ >>> from lp.bugs.externalbugtracker import ExternalBugTracker
97 >>> external_bugtracker = ExternalBugTracker('http://example.com/')
98 >>> chosen_bugtracker = external_bugtracker.getExternalBugTrackerToUse()
99 >>> chosen_bugtracker is external_bugtracker
100@@ -63,9 +62,11 @@
101 (ExternalBugTracker, bug_watches) tuples.
102
103 >>> from lp.bugs.externalbugtracker import (
104- ... Bugzilla, BugzillaAPI, BUG_TRACKER_CLASSES)
105- >>> from lp.bugs.interfaces.bugtracker import (
106- ... BugTrackerType)
107+ ... Bugzilla,
108+ ... BUG_TRACKER_CLASSES,
109+ ... )
110+ >>> from lp.bugs.externalbugtracker.bugzilla import BugzillaAPI
111+ >>> from lp.bugs.interfaces.bugtracker import BugTrackerType
112 >>> from lp.testing.factory import LaunchpadObjectFactory
113
114 >>> factory = LaunchpadObjectFactory()
115
116=== modified file 'lib/lp/bugs/externalbugtracker/__init__.py'
117--- lib/lp/bugs/externalbugtracker/__init__.py 2013-01-07 02:40:55 +0000
118+++ lib/lp/bugs/externalbugtracker/__init__.py 2016-07-13 10:12:00 +0000
119@@ -1,4 +1,4 @@
120-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
121+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
122 # GNU Affero General Public License version 3 (see the file LICENSE).
123
124 """__init__ module for the externalbugtracker package."""
125@@ -14,6 +14,7 @@
126 'DebBugs',
127 'DebBugsDatabaseNotFound',
128 'ExternalBugTracker',
129+ 'GitHub',
130 'InvalidBugId',
131 'LookupTree',
132 'Mantis',
133@@ -31,20 +32,43 @@
134 'get_external_bugtracker',
135 ]
136
137-from lp.bugs.externalbugtracker.base import *
138-from lp.bugs.externalbugtracker.bugzilla import *
139-from lp.bugs.externalbugtracker.debbugs import *
140-from lp.bugs.externalbugtracker.mantis import *
141-from lp.bugs.externalbugtracker.roundup import *
142-from lp.bugs.externalbugtracker.rt import *
143-from lp.bugs.externalbugtracker.sourceforge import *
144-from lp.bugs.externalbugtracker.trac import *
145+from lp.bugs.externalbugtracker.base import (
146+ BATCH_SIZE_UNLIMITED,
147+ BugNotFound,
148+ BugTrackerConnectError,
149+ BugWatchUpdateError,
150+ BugWatchUpdateWarning,
151+ ExternalBugTracker,
152+ InvalidBugId,
153+ LookupTree,
154+ PrivateRemoteBug,
155+ UnknownBugTrackerTypeError,
156+ UnknownRemoteStatusError,
157+ UnparsableBugData,
158+ UnparsableBugTrackerVersion,
159+ UnsupportedBugTrackerVersion,
160+ )
161+from lp.bugs.externalbugtracker.bugzilla import Bugzilla
162+from lp.bugs.externalbugtracker.debbugs import (
163+ DebBugs,
164+ DebBugsDatabaseNotFound,
165+ )
166+from lp.bugs.externalbugtracker.github import GitHub
167+from lp.bugs.externalbugtracker.mantis import (
168+ Mantis,
169+ MantisLoginHandler,
170+ )
171+from lp.bugs.externalbugtracker.roundup import Roundup
172+from lp.bugs.externalbugtracker.rt import RequestTracker
173+from lp.bugs.externalbugtracker.sourceforge import SourceForge
174+from lp.bugs.externalbugtracker.trac import Trac
175 from lp.bugs.interfaces.bugtracker import BugTrackerType
176
177
178 BUG_TRACKER_CLASSES = {
179 BugTrackerType.BUGZILLA: Bugzilla,
180 BugTrackerType.DEBBUGS: DebBugs,
181+ BugTrackerType.GITHUB: GitHub,
182 BugTrackerType.MANTIS: Mantis,
183 BugTrackerType.TRAC: Trac,
184 BugTrackerType.ROUNDUP: Roundup,
185
186=== modified file 'lib/lp/bugs/externalbugtracker/base.py'
187--- lib/lp/bugs/externalbugtracker/base.py 2015-07-08 16:05:11 +0000
188+++ lib/lp/bugs/externalbugtracker/base.py 2016-07-13 10:12:00 +0000
189@@ -1,4 +1,4 @@
190-# Copyright 2009-2013 Canonical Ltd. This software is licensed under the
191+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
192 # GNU Affero General Public License version 3 (see the file LICENSE).
193
194 """External bugtrackers."""
195@@ -14,6 +14,7 @@
196 'ExternalBugTracker',
197 'InvalidBugId',
198 'LookupTree',
199+ 'LP_USER_AGENT',
200 'PrivateRemoteBug',
201 'UnknownBugTrackerTypeError',
202 'UnknownRemoteImportanceError',
203@@ -236,7 +237,7 @@
204 def _getHeaders(self):
205 # For some reason, bugs.kde.org doesn't allow the regular urllib
206 # user-agent string (Python-urllib/2.x) to access their bugzilla.
207- return {'User-agent': LP_USER_AGENT, 'Host': self.basehost}
208+ return {'User-Agent': LP_USER_AGENT, 'Host': self.basehost}
209
210 def _fetchPage(self, page, data=None):
211 """Fetch a page from the remote server.
212
213=== added file 'lib/lp/bugs/externalbugtracker/github.py'
214--- lib/lp/bugs/externalbugtracker/github.py 1970-01-01 00:00:00 +0000
215+++ lib/lp/bugs/externalbugtracker/github.py 2016-07-13 10:12:00 +0000
216@@ -0,0 +1,264 @@
217+# Copyright 2016 Canonical Ltd. This software is licensed under the
218+# GNU Affero General Public License version 3 (see the file LICENSE).
219+
220+"""GitHub ExternalBugTracker utility."""
221+
222+__metaclass__ = type
223+__all__ = [
224+ 'BadGitHubURL',
225+ 'GitHub',
226+ 'GitHubRateLimit',
227+ 'IGitHubRateLimit',
228+ ]
229+
230+import httplib
231+import time
232+from urllib import urlencode
233+from urlparse import (
234+ urljoin,
235+ urlunsplit,
236+ )
237+
238+import pytz
239+import requests
240+from zope.component import getUtility
241+from zope.interface import Interface
242+
243+from lp.bugs.externalbugtracker import (
244+ BugTrackerConnectError,
245+ BugWatchUpdateError,
246+ ExternalBugTracker,
247+ UnknownRemoteStatusError,
248+ UnparsableBugTrackerVersion,
249+ )
250+from lp.bugs.externalbugtracker.base import LP_USER_AGENT
251+from lp.bugs.interfaces.bugtask import (
252+ BugTaskImportance,
253+ BugTaskStatus,
254+ )
255+from lp.bugs.interfaces.externalbugtracker import UNKNOWN_REMOTE_IMPORTANCE
256+from lp.services.config import config
257+from lp.services.database.isolation import ensure_no_transaction
258+from lp.services.webapp.url import urlsplit
259+
260+
261+class GitHubExceededRateLimit(BugWatchUpdateError):
262+
263+ def __init__(self, host, reset):
264+ self.host = host
265+ self.reset = reset
266+
267+ def __str__(self):
268+ return "Rate limit for %s exceeded (resets at %s)" % (
269+ self.host, time.ctime(self.reset))
270+
271+
272+class IGitHubRateLimit(Interface):
273+ """Interface for rate-limit tracking for the GitHub Issues API."""
274+
275+ def makeRequest(method, url, token=None, **kwargs):
276+ """Make a request, but only if the remote host's rate limit permits it.
277+
278+ :param method: The HTTP request method.
279+ :param url: The URL to request.
280+ :param token: If not None, an OAuth token to use as authentication
281+ to the remote host when asking it for the current rate limit.
282+ :return: A `requests.Response` object.
283+ :raises GitHubExceededRateLimit: if the rate limit was exceeded.
284+ """
285+
286+ def clearCache():
287+ """Forget any cached rate limits."""
288+
289+
290+class GitHubRateLimit:
291+ """Rate-limit tracking for the GitHub Issues API."""
292+
293+ def __init__(self):
294+ self.clearCache()
295+
296+ def _update(self, host, token=None):
297+ headers = {
298+ "User-Agent": LP_USER_AGENT,
299+ "Host": host,
300+ "Accept": "application/vnd.github.v3+json",
301+ }
302+ if token is not None:
303+ headers["Authorization"] = "token %s" % token
304+ url = "https://%s/rate_limit" % host
305+ try:
306+ response = requests.get(url, headers=headers)
307+ response.raise_for_status()
308+ self._limits[(host, token)] = response.json()["resources"]["core"]
309+ except requests.RequestException as e:
310+ raise BugTrackerConnectError(url, e)
311+
312+ @ensure_no_transaction
313+ def makeRequest(self, method, url, token=None, **kwargs):
314+ """See `IGitHubRateLimit`."""
315+ host = urlsplit(url).netloc
316+ if (host, token) not in self._limits:
317+ self._update(host, token=token)
318+ limit = self._limits[(host, token)]
319+ if not limit["remaining"]:
320+ raise GitHubExceededRateLimit(host, limit["reset"])
321+ response = requests.request(method, url, **kwargs)
322+ limit["remaining"] -= 1
323+ return response
324+
325+ def clearCache(self):
326+ """See `IGitHubRateLimit`."""
327+ self._limits = {}
328+
329+
330+class BadGitHubURL(UnparsableBugTrackerVersion):
331+ """The GitHub Issues URL is malformed."""
332+
333+
334+class GitHub(ExternalBugTracker):
335+ """An `ExternalBugTracker` for dealing with GitHub issues."""
336+
337+ # Avoid eating through our rate limit unnecessarily.
338+ batch_query_threshold = 1
339+
340+ def __init__(self, baseurl):
341+ _, host, path, query, fragment = urlsplit(baseurl)
342+ host = "api." + host
343+ path = path.rstrip("/")
344+ if not path.endswith("/issues"):
345+ raise BadGitHubURL(baseurl)
346+ path = "/repos" + path[:-len("/issues")]
347+ baseurl = urlunsplit(("https", host, path, query, fragment))
348+ super(GitHub, self).__init__(baseurl)
349+ self.cached_bugs = {}
350+
351+ @property
352+ def credentials(self):
353+ credentials_config = config["checkwatches.credentials"]
354+ # lazr.config.Section doesn't support get().
355+ try:
356+ token = credentials_config["%s.token" % self.basehost]
357+ except KeyError:
358+ token = None
359+ return {"token": token}
360+
361+ def getModifiedRemoteBugs(self, bug_ids, last_accessed):
362+ """See `IExternalBugTracker`."""
363+ modified_bugs = self.getRemoteBugBatch(
364+ bug_ids, last_accessed=last_accessed)
365+ self.cached_bugs.update(modified_bugs)
366+ return list(modified_bugs)
367+
368+ def getRemoteBug(self, bug_id):
369+ """See `ExternalBugTracker`."""
370+ bug_id = int(bug_id)
371+ if bug_id not in self.cached_bugs:
372+ self.cached_bugs[bug_id] = (
373+ self._getPage("issues/%s" % bug_id).json())
374+ return bug_id, self.cached_bugs[bug_id]
375+
376+ def getRemoteBugBatch(self, bug_ids, last_accessed=None):
377+ """See `ExternalBugTracker`."""
378+ # The GitHub API does not support exporting only a subset of bug IDs
379+ # as a batch. As a result, our caching is only effective if we have
380+ # cached *all* the requested bug IDs; this is the case when we're
381+ # being called on the result of getModifiedRemoteBugs, so it's still
382+ # a useful optimisation.
383+ bug_ids = [int(bug_id) for bug_id in bug_ids]
384+ bugs = {
385+ bug_id: self.cached_bugs[bug_id]
386+ for bug_id in bug_ids if bug_id in self.cached_bugs}
387+ if set(bugs) == set(bug_ids):
388+ return bugs
389+ params = [("state", "all")]
390+ if last_accessed is not None:
391+ since = last_accessed.astimezone(pytz.UTC).strftime(
392+ "%Y-%m-%dT%H:%M:%SZ")
393+ params.append(("since", since))
394+ page = "issues?%s" % urlencode(params)
395+ for remote_bug in self._getCollection(page):
396+ # We're only interested in the bug if it's one of the ones in
397+ # bug_ids.
398+ if remote_bug["id"] not in bug_ids:
399+ continue
400+ bugs[remote_bug["id"]] = remote_bug
401+ self.cached_bugs[remote_bug["id"]] = remote_bug
402+ return bugs
403+
404+ def getRemoteImportance(self, bug_id):
405+ """See `ExternalBugTracker`."""
406+ return UNKNOWN_REMOTE_IMPORTANCE
407+
408+ def getRemoteStatus(self, bug_id):
409+ """See `ExternalBugTracker`."""
410+ remote_bug = self.bugs[int(bug_id)]
411+ state = remote_bug["state"]
412+ labels = [label["name"] for label in remote_bug["labels"]]
413+ return " ".join([state] + labels)
414+
415+ def convertRemoteImportance(self, remote_importance):
416+ """See `IExternalBugTracker`."""
417+ return BugTaskImportance.UNKNOWN
418+
419+ def convertRemoteStatus(self, remote_status):
420+ """See `IExternalBugTracker`.
421+
422+ A GitHub status consists of the state followed by optional labels.
423+ """
424+ state = remote_status.split(" ", 1)[0]
425+ if state == "open":
426+ return BugTaskStatus.NEW
427+ elif state == "closed":
428+ return BugTaskStatus.FIXRELEASED
429+ else:
430+ raise UnknownRemoteStatusError(remote_status)
431+
432+ def _getHeaders(self, last_accessed=None):
433+ """See `ExternalBugTracker`."""
434+ headers = super(GitHub, self)._getHeaders()
435+ token = self.credentials["token"]
436+ if token is not None:
437+ headers["Authorization"] = "token %s" % token
438+ headers["Accept"] = "application/vnd.github.v3+json"
439+ if last_accessed is not None:
440+ headers["If-Modified-Since"] = (
441+ last_accessed.astimezone(pytz.UTC).strftime(
442+ "%a, %d %b %Y %H:%M:%S GMT"))
443+ return headers
444+
445+ def _getPage(self, page, last_accessed=None):
446+ """See `ExternalBugTracker`."""
447+ # We prefer to use requests here because it knows how to parse Link
448+ # headers. Note that this returns a `requests.Response`, not the
449+ # page data.
450+ try:
451+ response = getUtility(IGitHubRateLimit).makeRequest(
452+ "GET", urljoin(self.baseurl + "/", page),
453+ headers=self._getHeaders(last_accessed=last_accessed))
454+ response.raise_for_status()
455+ return response
456+ except requests.RequestException as e:
457+ raise BugTrackerConnectError(self.baseurl, e)
458+
459+ def _getCollection(self, base_page, last_accessed=None):
460+ """Yield each item from a batched remote collection.
461+
462+ If the collection has not been modified since `last_accessed`, yield
463+ no items.
464+ """
465+ page = base_page
466+ while page is not None:
467+ try:
468+ response = self._getPage(page, last_accessed=last_accessed)
469+ except BugTrackerConnectError as e:
470+ if (e.response is not None and
471+ e.response.status_code == httplib.NOT_MODIFIED):
472+ return
473+ else:
474+ raise
475+ for item in response.json():
476+ yield item
477+ if "next" in response.links:
478+ page = response.links["next"]["url"]
479+ else:
480+ page = None
481
482=== added file 'lib/lp/bugs/externalbugtracker/tests/test_github.py'
483--- lib/lp/bugs/externalbugtracker/tests/test_github.py 1970-01-01 00:00:00 +0000
484+++ lib/lp/bugs/externalbugtracker/tests/test_github.py 2016-07-13 10:12:00 +0000
485@@ -0,0 +1,375 @@
486+# Copyright 2016 Canonical Ltd. This software is licensed under the
487+# GNU Affero General Public License version 3 (see the file LICENSE).
488+
489+"""Tests for the GitHub Issues BugTracker."""
490+
491+from __future__ import absolute_import, print_function, unicode_literals
492+
493+__metaclass__ = type
494+
495+from datetime import datetime
496+import json
497+from urlparse import (
498+ parse_qs,
499+ urlunsplit,
500+ )
501+
502+from httmock import (
503+ HTTMock,
504+ urlmatch,
505+ )
506+import pytz
507+import transaction
508+from zope.component import getUtility
509+
510+from lp.app.interfaces.launchpad import ILaunchpadCelebrities
511+from lp.bugs.externalbugtracker import (
512+ BugTrackerConnectError,
513+ get_external_bugtracker,
514+ )
515+from lp.bugs.externalbugtracker.github import (
516+ BadGitHubURL,
517+ GitHub,
518+ GitHubExceededRateLimit,
519+ IGitHubRateLimit,
520+ )
521+from lp.bugs.interfaces.bugtask import BugTaskStatus
522+from lp.bugs.interfaces.bugtracker import BugTrackerType
523+from lp.bugs.interfaces.externalbugtracker import IExternalBugTracker
524+from lp.bugs.scripts.checkwatches import CheckwatchesMaster
525+from lp.services.log.logger import BufferLogger
526+from lp.testing import (
527+ TestCase,
528+ TestCaseWithFactory,
529+ verifyObject,
530+ )
531+from lp.testing.layers import (
532+ ZopelessDatabaseLayer,
533+ ZopelessLayer,
534+ )
535+
536+
537+class TestGitHubRateLimit(TestCase):
538+
539+ layer = ZopelessLayer
540+
541+ def setUp(self):
542+ super(TestGitHubRateLimit, self).setUp()
543+ self.rate_limit = getUtility(IGitHubRateLimit)
544+ self.addCleanup(self.rate_limit.clearCache)
545+
546+ @urlmatch(path=r"^/rate_limit$")
547+ def _rate_limit_handler(self, url, request):
548+ self.rate_limit_request = request
549+ self.rate_limit_headers = request.headers
550+ return {
551+ "status_code": 200,
552+ "content": {"resources": {"core": self.initial_rate_limit}},
553+ }
554+
555+ @urlmatch(path=r"^/$")
556+ def _target_handler(self, url, request):
557+ self.target_request = request
558+ return {"status_code": 200, "content": b"test"}
559+
560+ def test_makeRequest_no_token(self):
561+ self.initial_rate_limit = {
562+ "limit": 60, "remaining": 50, "reset": 1000000000}
563+ with HTTMock(self._rate_limit_handler, self._target_handler):
564+ response = self.rate_limit.makeRequest(
565+ "GET", "http://example.org/")
566+ self.assertNotIn("Authorization", self.rate_limit_headers)
567+ self.assertEqual(b"test", response.content)
568+ limit = self.rate_limit._limits[("example.org", None)]
569+ self.assertEqual(49, limit["remaining"])
570+ self.assertEqual(1000000000, limit["reset"])
571+
572+ limit["remaining"] = 0
573+ self.rate_limit_request = None
574+ with HTTMock(self._rate_limit_handler, self._target_handler):
575+ self.assertRaisesWithContent(
576+ GitHubExceededRateLimit,
577+ "Rate limit for example.org exceeded "
578+ "(resets at Sun Sep 9 07:16:40 2001)",
579+ self.rate_limit.makeRequest,
580+ "GET", "http://example.org/")
581+ self.assertIsNone(self.rate_limit_request)
582+ self.assertEqual(0, limit["remaining"])
583+
584+ def test_makeRequest_check_token(self):
585+ self.initial_rate_limit = {
586+ "limit": 5000, "remaining": 4000, "reset": 1000000000}
587+ with HTTMock(self._rate_limit_handler, self._target_handler):
588+ response = self.rate_limit.makeRequest(
589+ "GET", "http://example.org/", token="abc")
590+ self.assertEqual("token abc", self.rate_limit_headers["Authorization"])
591+ self.assertEqual(b"test", response.content)
592+ limit = self.rate_limit._limits[("example.org", "abc")]
593+ self.assertEqual(3999, limit["remaining"])
594+ self.assertEqual(1000000000, limit["reset"])
595+
596+ limit["remaining"] = 0
597+ self.rate_limit_request = None
598+ with HTTMock(self._rate_limit_handler, self._target_handler):
599+ self.assertRaisesWithContent(
600+ GitHubExceededRateLimit,
601+ "Rate limit for example.org exceeded "
602+ "(resets at Sun Sep 9 07:16:40 2001)",
603+ self.rate_limit.makeRequest,
604+ "GET", "http://example.org/", token="abc")
605+ self.assertIsNone(self.rate_limit_request)
606+ self.assertEqual(0, limit["remaining"])
607+
608+ def test_makeRequest_check_503(self):
609+ @urlmatch(path=r"^/rate_limit$")
610+ def rate_limit_handler(url, request):
611+ return {"status_code": 503}
612+
613+ with HTTMock(rate_limit_handler):
614+ self.assertRaises(
615+ BugTrackerConnectError, self.rate_limit.makeRequest,
616+ "GET", "http://example.org/")
617+
618+
619+class TestGitHub(TestCase):
620+
621+ layer = ZopelessLayer
622+
623+ def setUp(self):
624+ super(TestGitHub, self).setUp()
625+ self.addCleanup(getUtility(IGitHubRateLimit).clearCache)
626+ self.sample_bugs = [
627+ {"id": 1, "state": "open", "labels": []},
628+ {"id": 2, "state": "open", "labels": [{"name": "feature"}]},
629+ {"id": 3, "state": "open",
630+ "labels": [{"name": "feature"}, {"name": "ui"}]},
631+ {"id": 4, "state": "closed", "labels": []},
632+ {"id": 5, "state": "closed", "labels": [{"name": "feature"}]},
633+ ]
634+
635+ def test_implements_interface(self):
636+ self.assertTrue(verifyObject(
637+ IExternalBugTracker,
638+ GitHub("https://github.com/user/repository/issues")))
639+
640+ def test_requires_issues_url(self):
641+ self.assertRaises(
642+ BadGitHubURL, GitHub, "https://github.com/user/repository")
643+
644+ @urlmatch(path=r"^/rate_limit$")
645+ def _rate_limit_handler(self, url, request):
646+ self.rate_limit_request = request
647+ rate_limit = {"limit": 5000, "remaining": 4000, "reset": 1000000000}
648+ return {
649+ "status_code": 200,
650+ "content": {"resources": {"core": rate_limit}},
651+ }
652+
653+ def test_getRemoteBug(self):
654+ @urlmatch(path=r".*/issues/1$")
655+ def handler(url, request):
656+ self.request = request
657+ return {"status_code": 200, "content": self.sample_bugs[0]}
658+
659+ tracker = GitHub("https://github.com/user/repository/issues")
660+ with HTTMock(self._rate_limit_handler, handler):
661+ self.assertEqual(
662+ (1, self.sample_bugs[0]), tracker.getRemoteBug("1"))
663+ self.assertEqual(
664+ "https://api.github.com/repos/user/repository/issues/1",
665+ self.request.url)
666+
667+ @urlmatch(path=r".*/issues$")
668+ def _issues_handler(self, url, request):
669+ self.issues_request = request
670+ return {"status_code": 200, "content": json.dumps(self.sample_bugs)}
671+
672+ def test_getRemoteBugBatch(self):
673+ tracker = GitHub("https://github.com/user/repository/issues")
674+ with HTTMock(self._rate_limit_handler, self._issues_handler):
675+ self.assertEqual(
676+ {bug["id"]: bug for bug in self.sample_bugs[:2]},
677+ tracker.getRemoteBugBatch(["1", "2"]))
678+ self.assertEqual(
679+ "https://api.github.com/repos/user/repository/issues?state=all",
680+ self.issues_request.url)
681+
682+ def test_getRemoteBugBatch_last_accessed(self):
683+ tracker = GitHub("https://github.com/user/repository/issues")
684+ since = datetime(2015, 1, 1, 12, 0, 0, tzinfo=pytz.UTC)
685+ with HTTMock(self._rate_limit_handler, self._issues_handler):
686+ self.assertEqual(
687+ {bug["id"]: bug for bug in self.sample_bugs[:2]},
688+ tracker.getRemoteBugBatch(["1", "2"], last_accessed=since))
689+ self.assertEqual(
690+ "https://api.github.com/repos/user/repository/issues?"
691+ "state=all&since=2015-01-01T12%3A00%3A00Z",
692+ self.issues_request.url)
693+
694+ def test_getRemoteBugBatch_caching(self):
695+ tracker = GitHub("https://github.com/user/repository/issues")
696+ with HTTMock(self._rate_limit_handler, self._issues_handler):
697+ tracker.initializeRemoteBugDB(
698+ [str(bug["id"]) for bug in self.sample_bugs])
699+ self.issues_request = None
700+ self.assertEqual(
701+ {bug["id"]: bug for bug in self.sample_bugs[:2]},
702+ tracker.getRemoteBugBatch(["1", "2"]))
703+ self.assertIsNone(self.issues_request)
704+
705+ def test_getRemoteBugBatch_pagination(self):
706+ @urlmatch(path=r".*/issues")
707+ def handler(url, request):
708+ self.issues_requests.append(request)
709+ base_url = urlunsplit(list(url[:3]) + ["", ""])
710+ page = int(parse_qs(url.query).get("page", ["1"])[0])
711+ links = []
712+ if page != 3:
713+ links.append('<%s?page=%d>; rel="next"' % (base_url, page + 1))
714+ links.append('<%s?page=3>; rel="last"' % base_url)
715+ if page != 1:
716+ links.append('<%s?page=1>; rel="first"' % base_url)
717+ links.append('<%s?page=%d>; rel="prev"' % (base_url, page - 1))
718+ start = (page - 1) * 2
719+ end = page * 2
720+ return {
721+ "status_code": 200,
722+ "headers": {"Link": ", ".join(links)},
723+ "content": json.dumps(self.sample_bugs[start:end]),
724+ }
725+
726+ self.issues_requests = []
727+ tracker = GitHub("https://github.com/user/repository/issues")
728+ with HTTMock(self._rate_limit_handler, handler):
729+ self.assertEqual(
730+ {bug["id"]: bug for bug in self.sample_bugs},
731+ tracker.getRemoteBugBatch(
732+ [str(bug["id"]) for bug in self.sample_bugs]))
733+ expected_urls = [
734+ "https://api.github.com/repos/user/repository/issues?state=all",
735+ "https://api.github.com/repos/user/repository/issues?page=2",
736+ "https://api.github.com/repos/user/repository/issues?page=3",
737+ ]
738+ self.assertEqual(
739+ expected_urls, [request.url for request in self.issues_requests])
740+
741+ def test_status_open(self):
742+ self.sample_bugs = [
743+ {"id": 1, "state": "open", "labels": []},
744+ # Labels do not affect status, even if names collide.
745+ {"id": 2, "state": "open",
746+ "labels": [{"name": "feature"}, {"name": "closed"}]},
747+ ]
748+ tracker = GitHub("https://github.com/user/repository/issues")
749+ with HTTMock(self._rate_limit_handler, self._issues_handler):
750+ tracker.initializeRemoteBugDB(["1", "2"])
751+ remote_status = tracker.getRemoteStatus("1")
752+ self.assertEqual("open", remote_status)
753+ lp_status = tracker.convertRemoteStatus(remote_status)
754+ self.assertEqual(BugTaskStatus.NEW, lp_status)
755+ remote_status = tracker.getRemoteStatus("2")
756+ self.assertEqual("open feature closed", remote_status)
757+ lp_status = tracker.convertRemoteStatus(remote_status)
758+ self.assertEqual(BugTaskStatus.NEW, lp_status)
759+
760+ def test_status_closed(self):
761+ self.sample_bugs = [
762+ {"id": 1, "state": "closed", "labels": []},
763+ # Labels do not affect status, even if names collide.
764+ {"id": 2, "state": "closed",
765+ "labels": [{"name": "feature"}, {"name": "open"}]},
766+ ]
767+ tracker = GitHub("https://github.com/user/repository/issues")
768+ with HTTMock(self._rate_limit_handler, self._issues_handler):
769+ tracker.initializeRemoteBugDB(["1", "2"])
770+ remote_status = tracker.getRemoteStatus("1")
771+ self.assertEqual("closed", remote_status)
772+ lp_status = tracker.convertRemoteStatus(remote_status)
773+ self.assertEqual(BugTaskStatus.FIXRELEASED, lp_status)
774+ remote_status = tracker.getRemoteStatus("2")
775+ self.assertEqual("closed feature open", remote_status)
776+ lp_status = tracker.convertRemoteStatus(remote_status)
777+ self.assertEqual(BugTaskStatus.FIXRELEASED, lp_status)
778+
779+
780+class TestGitHubUpdateBugWatches(TestCaseWithFactory):
781+
782+ layer = ZopelessDatabaseLayer
783+
784+ @urlmatch(path=r"^/rate_limit$")
785+ def _rate_limit_handler(self, url, request):
786+ self.rate_limit_request = request
787+ rate_limit = {"limit": 5000, "remaining": 4000, "reset": 1000000000}
788+ return {
789+ "status_code": 200,
790+ "content": {"resources": {"core": rate_limit}},
791+ }
792+
793+ def test_process_one(self):
794+ remote_bug = {"id": 1234, "state": "open", "labels": []}
795+
796+ @urlmatch(path=r".*/issues/1234$")
797+ def handler(url, request):
798+ return {"status_code": 200, "content": remote_bug}
799+
800+ bug = self.factory.makeBug()
801+ bug_tracker = self.factory.makeBugTracker(
802+ base_url="https://github.com/user/repository/issues",
803+ bugtrackertype=BugTrackerType.GITHUB)
804+ bug.addWatch(
805+ bug_tracker, "1234", getUtility(ILaunchpadCelebrities).janitor)
806+ self.assertEqual(
807+ [("1234", None)],
808+ [(watch.remotebug, watch.remotestatus)
809+ for watch in bug_tracker.watches])
810+ transaction.commit()
811+ logger = BufferLogger()
812+ bug_watch_updater = CheckwatchesMaster(transaction, logger=logger)
813+ github = get_external_bugtracker(bug_tracker)
814+ with HTTMock(self._rate_limit_handler, handler):
815+ bug_watch_updater.updateBugWatches(github, bug_tracker.watches)
816+ self.assertEqual(
817+ "INFO Updating 1 watches for 1 bugs on "
818+ "https://api.github.com/repos/user/repository\n",
819+ logger.getLogBuffer())
820+ self.assertEqual(
821+ [("1234", BugTaskStatus.NEW)],
822+ [(watch.remotebug, github.convertRemoteStatus(watch.remotestatus))
823+ for watch in bug_tracker.watches])
824+
825+ def test_process_many(self):
826+ remote_bugs = [
827+ {"id": bug_id,
828+ "state": "open" if (bug_id % 2) == 0 else "closed",
829+ "labels": []}
830+ for bug_id in range(1000, 1010)]
831+
832+ @urlmatch(path=r".*/issues$")
833+ def handler(url, request):
834+ return {"status_code": 200, "content": json.dumps(remote_bugs)}
835+
836+ bug = self.factory.makeBug()
837+ bug_tracker = self.factory.makeBugTracker(
838+ base_url="https://github.com/user/repository/issues",
839+ bugtrackertype=BugTrackerType.GITHUB)
840+ for remote_bug in remote_bugs:
841+ bug.addWatch(
842+ bug_tracker, str(remote_bug["id"]),
843+ getUtility(ILaunchpadCelebrities).janitor)
844+ transaction.commit()
845+ logger = BufferLogger()
846+ bug_watch_updater = CheckwatchesMaster(transaction, logger=logger)
847+ github = get_external_bugtracker(bug_tracker)
848+ with HTTMock(self._rate_limit_handler, handler):
849+ bug_watch_updater.updateBugWatches(github, bug_tracker.watches)
850+ self.assertEqual(
851+ "INFO Updating 10 watches for 10 bugs on "
852+ "https://api.github.com/repos/user/repository\n",
853+ logger.getLogBuffer())
854+ self.assertContentEqual(
855+ [(str(bug_id), BugTaskStatus.NEW)
856+ for bug_id in (1000, 1002, 1004, 1006, 1008)] +
857+ [(str(bug_id), BugTaskStatus.FIXRELEASED)
858+ for bug_id in (1001, 1003, 1005, 1007, 1009)],
859+ [(watch.remotebug, github.convertRemoteStatus(watch.remotestatus))
860+ for watch in bug_tracker.watches])
861
862=== modified file 'lib/lp/bugs/interfaces/bugtracker.py'
863--- lib/lp/bugs/interfaces/bugtracker.py 2013-05-02 18:55:32 +0000
864+++ lib/lp/bugs/interfaces/bugtracker.py 2016-07-13 10:12:00 +0000
865@@ -1,4 +1,4 @@
866-# Copyright 2009-2013 Canonical Ltd. This software is licensed under the
867+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
868 # GNU Affero General Public License version 3 (see the file LICENSE).
869
870 """Bug tracker interfaces."""
871@@ -186,6 +186,12 @@
872 Google.
873 """)
874
875+ GITHUB = DBItem(12, """
876+ GitHub Issues
877+
878+ The issue tracker for projects hosted on GitHub.
879+ """)
880+
881
882 # A list of the BugTrackerTypes that don't need a remote product to be
883 # able to return a bug filing URL. We use a whitelist rather than a
884@@ -193,6 +199,7 @@
885 # a remote product is required. This saves us from presenting
886 # embarrassingly useless URLs to users.
887 SINGLE_PRODUCT_BUGTRACKERTYPES = [
888+ BugTrackerType.GITHUB,
889 BugTrackerType.GOOGLE_CODE,
890 BugTrackerType.MANTIS,
891 BugTrackerType.PHPPROJECT,
892
893=== modified file 'lib/lp/bugs/model/bugtracker.py'
894--- lib/lp/bugs/model/bugtracker.py 2015-07-08 16:05:11 +0000
895+++ lib/lp/bugs/model/bugtracker.py 2016-07-13 10:12:00 +0000
896@@ -1,4 +1,4 @@
897-# Copyright 2009-2013 Canonical Ltd. This software is licensed under the
898+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
899 # GNU Affero General Public License version 3 (see the file LICENSE).
900
901 __metaclass__ = type
902@@ -331,6 +331,8 @@
903 BugTrackerType.BUGZILLA: (
904 "%(base_url)s/enter_bug.cgi?product=%(remote_product)s"
905 "&short_desc=%(summary)s&long_desc=%(description)s"),
906+ BugTrackerType.GITHUB: (
907+ "%(base_url)s/new?title=%(summary)s&body=%(description)s"),
908 BugTrackerType.GOOGLE_CODE: (
909 "%(base_url)s/entry?summary=%(summary)s&"
910 "comment=%(description)s"),
911@@ -360,6 +362,9 @@
912 BugTrackerType.BUGZILLA: (
913 "%(base_url)s/query.cgi?product=%(remote_product)s"
914 "&short_desc=%(summary)s"),
915+ BugTrackerType.GITHUB: (
916+ "%(base_url)s?utf8=%%E2%%9C%%93"
917+ "&q=is%%3Aissue%%20is%%3Aopen%%20%(summary)s"),
918 BugTrackerType.GOOGLE_CODE: "%(base_url)s/list?q=%(summary)s",
919 BugTrackerType.DEBBUGS: (
920 "%(base_url)s/cgi-bin/search.cgi?phrase=%(summary)s"
921
922=== modified file 'lib/lp/bugs/model/bugwatch.py'
923--- lib/lp/bugs/model/bugwatch.py 2015-07-08 16:05:11 +0000
924+++ lib/lp/bugs/model/bugwatch.py 2016-07-13 10:12:00 +0000
925@@ -1,4 +1,4 @@
926-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
927+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
928 # GNU Affero General Public License version 3 (see the file LICENSE).
929
930 __metaclass__ = type
931@@ -81,6 +81,7 @@
932 BUG_TRACKER_URL_FORMATS = {
933 BugTrackerType.BUGZILLA: 'show_bug.cgi?id=%s',
934 BugTrackerType.DEBBUGS: 'cgi-bin/bugreport.cgi?bug=%s',
935+ BugTrackerType.GITHUB: '%s',
936 BugTrackerType.GOOGLE_CODE: 'detail?id=%s',
937 BugTrackerType.MANTIS: 'view.php?id=%s',
938 BugTrackerType.ROUNDUP: 'issue%s',
939@@ -388,6 +389,7 @@
940 BugTrackerType.BUGZILLA: self.parseBugzillaURL,
941 BugTrackerType.DEBBUGS: self.parseDebbugsURL,
942 BugTrackerType.EMAILADDRESS: self.parseEmailAddressURL,
943+ BugTrackerType.GITHUB: self.parseGitHubURL,
944 BugTrackerType.GOOGLE_CODE: self.parseGoogleCodeURL,
945 BugTrackerType.MANTIS: self.parseMantisURL,
946 BugTrackerType.PHPPROJECT: self.parsePHPProjectURL,
947@@ -688,6 +690,18 @@
948 base_url = urlunsplit((scheme, host, tracker_path, '', ''))
949 return base_url, remote_bug
950
951+ def parseGitHubURL(self, scheme, host, path, query):
952+ """Extract a GitHub Issues base URL and bug ID."""
953+ if host != 'github.com':
954+ return None
955+ match = re.match(r'(.*/issues)/(\d+)$', path)
956+ if not match:
957+ return None
958+ base_path = match.group(1)
959+ remote_bug = match.group(2)
960+ base_url = urlunsplit((scheme, host, base_path, '', ''))
961+ return base_url, remote_bug
962+
963 def extractBugTrackerAndBug(self, url):
964 """See `IBugWatchSet`."""
965 for trackertype, parse_func in (
966
967=== modified file 'lib/lp/bugs/stories/bugtracker/xx-bugtracker.txt'
968--- lib/lp/bugs/stories/bugtracker/xx-bugtracker.txt 2015-06-27 04:10:49 +0000
969+++ lib/lp/bugs/stories/bugtracker/xx-bugtracker.txt 2016-07-13 10:12:00 +0000
970@@ -40,6 +40,7 @@
971 Savane
972 PHP Project Bugtracker
973 Google Code
974+ GitHub Issues
975
976 The bug tracker name is used in URLs and certain characters (like '!')
977 aren't allowed.
978
979=== modified file 'lib/lp/bugs/tests/test_bugwatch.py'
980--- lib/lp/bugs/tests/test_bugwatch.py 2015-10-15 14:09:50 +0000
981+++ lib/lp/bugs/tests/test_bugwatch.py 2016-07-13 10:12:00 +0000
982@@ -1,4 +1,4 @@
983-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
984+# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
985 # GNU Affero General Public License version 3 (see the file LICENSE).
986
987 """Tests for BugWatchSet."""
988@@ -10,12 +10,15 @@
989 timedelta,
990 )
991 import re
992-import unittest
993 from urlparse import urlunsplit
994
995 from lazr.lifecycle.snapshot import Snapshot
996 from pytz import utc
997 from storm.store import Store
998+from testscenarios import (
999+ load_tests_apply_scenarios,
1000+ WithScenarios,
1001+ )
1002 import transaction
1003 from zope.component import getUtility
1004 from zope.security.interfaces import Unauthorized
1005@@ -50,6 +53,7 @@
1006 ANONYMOUS,
1007 login,
1008 login_person,
1009+ TestCase,
1010 TestCaseWithFactory,
1011 )
1012 from lp.testing.dbuser import switch_dbuser
1013@@ -61,8 +65,111 @@
1014 from lp.testing.sampledata import ADMIN_EMAIL
1015
1016
1017-class ExtractBugTrackerAndBugTestBase:
1018- """Test base for testing BugWatchSet.extractBugTrackerAndBug."""
1019+class ExtractBugTrackerAndBugTest(WithScenarios, TestCase):
1020+ """Test BugWatchSet.extractBugTrackerAndBug."""
1021+
1022+ scenarios = [
1023+ ('Mantis', {
1024+ 'bugtracker_type': BugTrackerType.MANTIS,
1025+ 'bug_url': 'http://some.host/bugs/view.php?id=3224',
1026+ 'base_url': 'http://some.host/bugs/',
1027+ 'bug_id': '3224',
1028+ }),
1029+ ('Bugzilla', {
1030+ 'bugtracker_type': BugTrackerType.BUGZILLA,
1031+ 'bug_url': 'http://some.host/bugs/show_bug.cgi?id=3224',
1032+ 'base_url': 'http://some.host/bugs/',
1033+ 'bug_id': '3224',
1034+ }),
1035+ # Issuezilla is practically the same as Bugzilla, so we treat it as
1036+ # a normal BUGZILLA type.
1037+ ('Issuezilla', {
1038+ 'bugtracker_type': BugTrackerType.BUGZILLA,
1039+ 'bug_url': 'http://some.host/bugs/show_bug.cgi?issue=3224',
1040+ 'base_url': 'http://some.host/bugs/',
1041+ 'bug_id': '3224',
1042+ }),
1043+ ('RoundUp', {
1044+ 'bugtracker_type': BugTrackerType.ROUNDUP,
1045+ 'bug_url': 'http://some.host/some/path/issue377',
1046+ 'base_url': 'http://some.host/some/path/',
1047+ 'bug_id': '377',
1048+ }),
1049+ ('Trac', {
1050+ 'bugtracker_type': BugTrackerType.TRAC,
1051+ 'bug_url': 'http://some.host/some/path/ticket/42',
1052+ 'base_url': 'http://some.host/some/path/',
1053+ 'bug_id': '42',
1054+ }),
1055+ ('Debbugs', {
1056+ 'bugtracker_type': BugTrackerType.DEBBUGS,
1057+ 'bug_url': (
1058+ 'http://some.host/some/path/cgi-bin/bugreport.cgi?bug=42'),
1059+ 'base_url': 'http://some.host/some/path/',
1060+ 'bug_id': '42',
1061+ }),
1062+ ('DebbugsShorthand', {
1063+ 'bugtracker_type': BugTrackerType.DEBBUGS,
1064+ 'bug_url': 'http://bugs.debian.org/42',
1065+ 'base_url': 'http://bugs.debian.org/',
1066+ 'bug_id': '42',
1067+ 'already_registered': True,
1068+ }),
1069+ # SourceForge-like URLs, though not actually SourceForge itself.
1070+ ('XForge', {
1071+ 'bugtracker_type': BugTrackerType.SOURCEFORGE,
1072+ 'bug_url': (
1073+ 'http://gforge.example.com/tracker/index.php'
1074+ '?func=detail&aid=90812&group_id=84122&atid=575154'),
1075+ 'base_url': 'http://gforge.example.com/',
1076+ 'bug_id': '90812',
1077+ }),
1078+ ('RT', {
1079+ 'bugtracker_type': BugTrackerType.RT,
1080+ 'bug_url': 'http://some.host/Ticket/Display.html?id=2379',
1081+ 'base_url': 'http://some.host/',
1082+ 'bug_id': '2379',
1083+ }),
1084+ ('CPAN', {
1085+ 'bugtracker_type': BugTrackerType.RT,
1086+ 'bug_url': 'http://rt.cpan.org/Public/Bug/Display.html?id=2379',
1087+ 'base_url': 'http://rt.cpan.org/',
1088+ 'bug_id': '2379',
1089+ }),
1090+ ('Savannah', {
1091+ 'bugtracker_type': BugTrackerType.SAVANE,
1092+ 'bug_url': 'http://savannah.gnu.org/bugs/?22003',
1093+ 'base_url': 'http://savannah.gnu.org/',
1094+ 'bug_id': '22003',
1095+ 'already_registered': True,
1096+ }),
1097+ ('Savane', {
1098+ 'bugtracker_type': BugTrackerType.SAVANE,
1099+ 'bug_url': 'http://savane.example.com/bugs/?12345',
1100+ 'base_url': 'http://savane.example.com/',
1101+ 'bug_id': '12345',
1102+ }),
1103+ ('PHPProject', {
1104+ 'bugtracker_type': BugTrackerType.PHPPROJECT,
1105+ 'bug_url': 'http://phptracker.example.com/bug.php?id=12345',
1106+ 'base_url': 'http://phptracker.example.com/',
1107+ 'bug_id': '12345',
1108+ }),
1109+ ('GoogleCode', {
1110+ 'bugtracker_type': BugTrackerType.GOOGLE_CODE,
1111+ 'bug_url': (
1112+ 'http://code.google.com/p/myproject/issues/detail?id=12345'),
1113+ 'base_url': 'http://code.google.com/p/myproject/issues',
1114+ 'bug_id': '12345',
1115+ }),
1116+ ('GitHub', {
1117+ 'bugtracker_type': BugTrackerType.GITHUB,
1118+ 'bug_url': 'https://github.com/user/repository/issues/12345',
1119+ 'base_url': 'https://github.com/user/repository/issues',
1120+ 'bug_id': '12345',
1121+ }),
1122+ ]
1123+
1124 layer = LaunchpadFunctionalLayer
1125
1126 # A URL to an unregistered bug tracker.
1127@@ -77,7 +184,11 @@
1128 # The bug id in the sample bug_url.
1129 bug_id = None
1130
1131+ # True if the bug tracker is already registered in sampledata.
1132+ already_registered = False
1133+
1134 def setUp(self):
1135+ super(ExtractBugTrackerAndBugTest, self).setUp()
1136 login(ANONYMOUS)
1137 self.bugwatch_set = getUtility(IBugWatchSet)
1138 self.bugtracker_set = getUtility(IBugTrackerSet)
1139@@ -108,8 +219,9 @@
1140 # A NoBugTrackerFound exception is raised if extractBugTrackerAndBug
1141 # can extract a base URL and bug id from the URL but there's no
1142 # such bug tracker registered in Launchpad.
1143- self.failUnless(
1144- self.bugtracker_set.queryByBaseURL(self.base_url) is None)
1145+ if self.already_registered:
1146+ return
1147+ self.assertIsNone(self.bugtracker_set.queryByBaseURL(self.base_url))
1148 try:
1149 bugtracker, bug = self.bugwatch_set.extractBugTrackerAndBug(
1150 self.bug_url)
1151@@ -132,86 +244,7 @@
1152 self.bugwatch_set.extractBugTrackerAndBug, invalid_url)
1153
1154
1155-class MantisExtractBugTrackerAndBugTest(
1156- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1157- """Ensure BugWatchSet.extractBugTrackerAndBug works with Mantis URLs."""
1158-
1159- bugtracker_type = BugTrackerType.MANTIS
1160- bug_url = 'http://some.host/bugs/view.php?id=3224'
1161- base_url = 'http://some.host/bugs/'
1162- bug_id = '3224'
1163-
1164-
1165-class BugzillaExtractBugTrackerAndBugTest(
1166- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1167- """Ensure BugWatchSet.extractBugTrackerAndBug works with Bugzilla URLs."""
1168-
1169- bugtracker_type = BugTrackerType.BUGZILLA
1170- bug_url = 'http://some.host/bugs/show_bug.cgi?id=3224'
1171- base_url = 'http://some.host/bugs/'
1172- bug_id = '3224'
1173-
1174-
1175-class IssuezillaExtractBugTrackerAndBugTest(
1176- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1177- """Ensure BugWatchSet.extractBugTrackerAndBug works with Issuezilla.
1178-
1179- Issuezilla is practically the same as Buzilla, so we treat it as a
1180- normal BUGZILLA type.
1181- """
1182-
1183- bugtracker_type = BugTrackerType.BUGZILLA
1184- bug_url = 'http://some.host/bugs/show_bug.cgi?issue=3224'
1185- base_url = 'http://some.host/bugs/'
1186- bug_id = '3224'
1187-
1188-
1189-class RoundUpExtractBugTrackerAndBugTest(
1190- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1191- """Ensure BugWatchSet.extractBugTrackerAndBug works with RoundUp URLs."""
1192-
1193- bugtracker_type = BugTrackerType.ROUNDUP
1194- bug_url = 'http://some.host/some/path/issue377'
1195- base_url = 'http://some.host/some/path/'
1196- bug_id = '377'
1197-
1198-
1199-class TracExtractBugTrackerAndBugTest(
1200- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1201- """Ensure BugWatchSet.extractBugTrackerAndBug works with Trac URLs."""
1202-
1203- bugtracker_type = BugTrackerType.TRAC
1204- bug_url = 'http://some.host/some/path/ticket/42'
1205- base_url = 'http://some.host/some/path/'
1206- bug_id = '42'
1207-
1208-
1209-class DebbugsExtractBugTrackerAndBugTest(
1210- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1211- """Ensure BugWatchSet.extractBugTrackerAndBug works with Debbugs URLs."""
1212-
1213- bugtracker_type = BugTrackerType.DEBBUGS
1214- bug_url = 'http://some.host/some/path/cgi-bin/bugreport.cgi?bug=42'
1215- base_url = 'http://some.host/some/path/'
1216- bug_id = '42'
1217-
1218-
1219-class DebbugsExtractBugTrackerAndBugShorthandTest(
1220- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1221- """Ensure extractBugTrackerAndBug works for short Debbugs URLs."""
1222-
1223- bugtracker_type = BugTrackerType.DEBBUGS
1224- bug_url = 'http://bugs.debian.org/42'
1225- base_url = 'http://bugs.debian.org/'
1226- bug_id = '42'
1227-
1228- def test_unregistered_tracker_url(self):
1229- # bugs.debian.org is already registered, so no dice.
1230- pass
1231-
1232-
1233-class SFExtractBugTrackerAndBugTest(
1234- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1235+class SFExtractBugTrackerAndBugTest(ExtractBugTrackerAndBugTest):
1236 """Ensure BugWatchSet.extractBugTrackerAndBug works with SF URLs.
1237
1238 We have only one SourceForge tracker registered in Launchpad, so we
1239@@ -219,17 +252,31 @@
1240 bug id.
1241 """
1242
1243- bugtracker_type = BugTrackerType.SOURCEFORGE
1244- bug_url = (
1245- 'http://sourceforge.net/tracker/index.php'
1246- '?func=detail&aid=1568562&group_id=84122&atid=575154')
1247- base_url = 'http://sourceforge.net/'
1248- bug_id = '1568562'
1249+ scenarios = [
1250+ # We have only one SourceForge tracker registered in Launchpad, so
1251+ # we don't care about the aid and group_id, only about atid which is
1252+ # the bug id.
1253+ ('SourceForge', {
1254+ 'bugtracker_type': BugTrackerType.SOURCEFORGE,
1255+ 'bug_url': (
1256+ 'http://sourceforge.net/tracker/index.php'
1257+ '?func=detail&aid=1568562&group_id=84122&atid=575154'),
1258+ 'base_url': 'http://sourceforge.net/',
1259+ 'bug_id': '1568562',
1260+ }),
1261+ # New SF tracker URLs.
1262+ ('SourceForgeTracker2', {
1263+ 'bugtracker_type': BugTrackerType.SOURCEFORGE,
1264+ 'bug_url': (
1265+ 'http://sourceforge.net/tracker2/'
1266+ '?func=detail&aid=1568562&group_id=84122&atid=575154'),
1267+ 'base_url': 'http://sourceforge.net/',
1268+ 'bug_id': '1568562',
1269+ }),
1270+ ]
1271
1272- def test_unregistered_tracker_url(self):
1273- # The SourceForge tracker is always registered, so this test
1274- # doesn't make sense for SourceForge URLs.
1275- pass
1276+ # The SourceForge tracker is always registered.
1277+ already_registered = True
1278
1279 def test_aliases(self):
1280 """Test that parsing SourceForge URLs works with the SF aliases."""
1281@@ -260,82 +307,11 @@
1282 self.base_url = original_base_url
1283
1284
1285-class SFTracker2ExtractBugTrackerAndBugTest(SFExtractBugTrackerAndBugTest):
1286- """Ensure extractBugTrackerAndBug works for new SF tracker URLs."""
1287-
1288- bugtracker_type = BugTrackerType.SOURCEFORGE
1289- bug_url = (
1290- 'http://sourceforge.net/tracker2/'
1291- '?func=detail&aid=1568562&group_id=84122&atid=575154')
1292- base_url = 'http://sourceforge.net/'
1293- bug_id = '1568562'
1294-
1295-
1296-class XForgeExtractBugTrackerAndBugTest(
1297- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1298- """Ensure extractBugTrackerAndBug works with SourceForge-like URLs.
1299- """
1300-
1301- bugtracker_type = BugTrackerType.SOURCEFORGE
1302- bug_url = (
1303- 'http://gforge.example.com/tracker/index.php'
1304- '?func=detail&aid=90812&group_id=84122&atid=575154')
1305- base_url = 'http://gforge.example.com/'
1306- bug_id = '90812'
1307-
1308-
1309-class RTExtractBugTrackerAndBugTest(
1310- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1311- """Ensure BugWatchSet.extractBugTrackerAndBug works with RT URLs."""
1312-
1313- bugtracker_type = BugTrackerType.RT
1314- bug_url = 'http://some.host/Ticket/Display.html?id=2379'
1315- base_url = 'http://some.host/'
1316- bug_id = '2379'
1317-
1318-
1319-class CpanExtractBugTrackerAndBugTest(
1320- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1321- """Ensure BugWatchSet.extractBugTrackerAndBug works with CPAN URLs."""
1322-
1323- bugtracker_type = BugTrackerType.RT
1324- bug_url = 'http://rt.cpan.org/Public/Bug/Display.html?id=2379'
1325- base_url = 'http://rt.cpan.org/'
1326- bug_id = '2379'
1327-
1328-
1329-class SavannahExtractBugTrackerAndBugTest(
1330- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1331- """Ensure BugWatchSet.extractBugTrackerAndBug works with Savannah URLs.
1332- """
1333-
1334- bugtracker_type = BugTrackerType.SAVANE
1335- bug_url = 'http://savannah.gnu.org/bugs/?22003'
1336- base_url = 'http://savannah.gnu.org/'
1337- bug_id = '22003'
1338-
1339- def test_unregistered_tracker_url(self):
1340- # The Savannah tracker is always registered, so this test
1341- # doesn't make sense for Savannah URLs.
1342- pass
1343-
1344-
1345-class SavaneExtractBugTrackerAndBugTest(
1346- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1347- """Ensure BugWatchSet.extractBugTrackerAndBug works with Savane URLs.
1348- """
1349-
1350- bugtracker_type = BugTrackerType.SAVANE
1351- bug_url = 'http://savane.example.com/bugs/?12345'
1352- base_url = 'http://savane.example.com/'
1353- bug_id = '12345'
1354-
1355-
1356-class EmailAddressExtractBugTrackerAndBugTest(
1357- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1358+class EmailAddressExtractBugTrackerAndBugTest(ExtractBugTrackerAndBugTest):
1359 """Ensure BugWatchSet.extractBugTrackerAndBug works with email addresses.
1360 """
1361
1362+ scenarios = None
1363 bugtracker_type = BugTrackerType.EMAILADDRESS
1364 bug_url = 'mailto:foo.bar@example.com'
1365 base_url = 'mailto:foo.bar@example.com'
1366@@ -353,28 +329,6 @@
1367 pass
1368
1369
1370-class PHPProjectBugTrackerExtractBugTrackerAndBugTest(
1371- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1372- """Ensure BugWatchSet.extractBugTrackerAndBug works with PHP bug URLs.
1373- """
1374-
1375- bugtracker_type = BugTrackerType.PHPPROJECT
1376- bug_url = 'http://phptracker.example.com/bug.php?id=12345'
1377- base_url = 'http://phptracker.example.com/'
1378- bug_id = '12345'
1379-
1380-
1381-class GoogleCodeBugTrackerExtractBugTrackerAndBugTest(
1382- ExtractBugTrackerAndBugTestBase, unittest.TestCase):
1383- """Ensure BugWatchSet.extractBugTrackerAndBug works for Google Code URLs.
1384- """
1385-
1386- bugtracker_type = BugTrackerType.GOOGLE_CODE
1387- bug_url = 'http://code.google.com/p/myproject/issues/detail?id=12345'
1388- base_url = 'http://code.google.com/p/myproject/issues'
1389- bug_id = '12345'
1390-
1391-
1392 class TestBugWatch(TestCaseWithFactory):
1393
1394 layer = LaunchpadZopelessLayer
1395@@ -758,3 +712,6 @@
1396 login_person(lp_dev)
1397 self.bug_watch.reset()
1398 self._assertBugWatchHasBeenChanged()
1399+
1400+
1401+load_tests = load_tests_apply_scenarios