Merge ~cjwatson/launchpad:dateutil.tz into launchpad:master

[Colin Watson]

Remove direct dependencies on pytz

`dateutil.tz` is a better fit for Python's modern timezone provider interface, and has fewer footguns as a result.

The only significant downside is that we have to reimplement something similar to `pytz.common_timezones` for use by our timezone vocabulary. Fortunately this isn't too difficult.

[Jürgen Gmach (jugmac00)]

review: Approve

diff --git a/lib/lp/app/browser/tales.py b/lib/lp/app/browser/tales.py
index 0e0b0e5..44781e8 100644
--- a/lib/lp/app/browser/tales.py
+++ b/lib/lp/app/browser/tales.py
@@ -12,7 +12,7 @@ from email.utils import formatdate, mktime_tz
 from textwrap import dedent
 from urllib.parse import quote
 
-import pytz
+from dateutil import tz
 from lazr.restful.utils import get_current_browser_request
 from lazr.uri import URI
 from zope.browserpage import ViewPageTemplateFile
@@ -1293,7 +1293,7 @@ class PersonFormatterAPI(ObjectFormatterAPI):
     def local_time(self):
         """Return the local time for this person."""
         time_zone = self._context.time_zone
-        dt = datetime.now(pytz.timezone(time_zone))
+        dt = datetime.now(tz.gettz(time_zone))
         return "%s %s" % (dt.strftime("%T"), tzname(dt))
 
     def url(self, view_name=None, rootsite="mainsite"):
diff --git a/lib/lp/app/widgets/date.py b/lib/lp/app/widgets/date.py
index e024dfa..ce27881 100644
--- a/lib/lp/app/widgets/date.py
+++ b/lib/lp/app/widgets/date.py
@@ -19,7 +19,7 @@ __all__ = [
 
 from datetime import datetime, timezone, tzinfo
 
-import pytz
+from dateutil import tz
 from zope.browserpage import ViewPageTemplateFile
 from zope.component import getUtility
 from zope.datetime import DateTimeError, parse
@@ -217,14 +217,23 @@ class DateTimeWidget(TextWidget):
           >>> widget.required_time_zone_name = "Africa/Maseru"
           >>> print(widget.time_zone_name)
           Africa/Maseru
-          >>> print(widget.time_zone)
-          Africa/Maseru
+          >>> print(widget.time_zone)  # doctest: +ELLIPSIS
+          tzfile('.../Africa/Maseru')
+
+        When the required_time_zone_name is invalid, we fall back to UTC.
+
+          >>> widget.required_time_zone_name = "Some/Nonsense"
+          >>> print(widget.time_zone_name)
+          Some/Nonsense
+          >>> print(repr(widget.time_zone))
+          datetime.timezone.utc
 
         """
         if self.time_zone_name == "UTC":
             return timezone.utc
         else:
-            return pytz.timezone(self.time_zone_name)
+            zone = tz.gettz(self.time_zone_name)
+            return zone if zone is not None else timezone.utc
 
     def _align_date_constraints_with_time_zone(self):
         """Ensure that from_date and to_date use the widget time zone."""
@@ -232,22 +241,14 @@ class DateTimeWidget(TextWidget):
             if self.from_date.tzinfo is None:
                 # Timezone-naive constraint is interpreted as being in the
                 # widget time zone.
-                if hasattr(self.time_zone, "localize"):  # pytz
-                    self.from_date = self.time_zone.localize(self.from_date)
-                else:
-                    self.from_date = self.from_date.replace(
-                        tzinfo=self.time_zone
-                    )
+                self.from_date = self.from_date.replace(tzinfo=self.time_zone)
             else:
                 self.from_date = self.from_date.astimezone(self.time_zone)
         if isinstance(self.to_date, datetime):
             if self.to_date.tzinfo is None:
                 # Timezone-naive constraint is interpreted as being in the
                 # widget time zone.
-                if hasattr(self.time_zone, "localize"):  # pytz
-                    self.to_date = self.time_zone.localize(self.to_date)
-                else:
-                    self.to_date = self.to_date.replace(tzinfo=self.time_zone)
+                self.to_date = self.to_date.replace(tzinfo=self.time_zone)
             else:
                 self.to_date = self.to_date.astimezone(self.time_zone)
 
@@ -426,10 +427,7 @@ class DateTimeWidget(TextWidget):
             dt = datetime(year, month, day, hour, minute, int(second), micro)
         except (DateTimeError, ValueError, IndexError) as v:
             raise ConversionError("Invalid date value", v)
-        if hasattr(self.time_zone, "localize"):  # pytz
-            return self.time_zone.localize(dt)
-        else:
-            return dt.replace(tzinfo=self.time_zone)
+        return dt.replace(tzinfo=self.time_zone)
 
     def _toFormValue(self, value):
         """Convert a date to its string representation.
diff --git a/lib/lp/blueprints/browser/sprint.py b/lib/lp/blueprints/browser/sprint.py
index 41148c0..debe896 100644
--- a/lib/lp/blueprints/browser/sprint.py
+++ b/lib/lp/blueprints/browser/sprint.py
@@ -27,7 +27,7 @@ import io
 from collections import defaultdict
 from typing import List
 
-import pytz
+from dateutil import tz
 from lazr.restful.utils import smartquote
 from zope.component import getUtility
 from zope.formlib.widget import CustomWidgetFactory
@@ -190,7 +190,7 @@ class SprintView(HasSpecificationsView):
     def initialize(self):
         self.notices = []
         self.latest_specs_limit = 5
-        self.tzinfo = pytz.timezone(self.context.time_zone)
+        self.tzinfo = tz.gettz(self.context.time_zone)
 
     def attendance(self):
         """establish if this user is attending"""
@@ -246,14 +246,18 @@ class SprintView(HasSpecificationsView):
     @property
     def local_start(self):
         """The sprint start time, in the local time zone, as text."""
-        tz = pytz.timezone(self.context.time_zone)
-        return self._formatLocal(self.context.time_starts.astimezone(tz))
+        return self._formatLocal(
+            self.context.time_starts.astimezone(
+                tz.gettz(self.context.time_zone)
+            )
+        )
 
     @property
     def local_end(self):
         """The sprint end time, in the local time zone, as text."""
-        tz = pytz.timezone(self.context.time_zone)
-        return self._formatLocal(self.context.time_ends.astimezone(tz))
+        return self._formatLocal(
+            self.context.time_ends.astimezone(tz.gettz(self.context.time_zone))
+        )
 
 
 class SprintAddView(LaunchpadFormView):
diff --git a/lib/lp/blueprints/browser/sprintattendance.py b/lib/lp/blueprints/browser/sprintattendance.py
index 194f340..baeea2f 100644
--- a/lib/lp/blueprints/browser/sprintattendance.py
+++ b/lib/lp/blueprints/browser/sprintattendance.py
@@ -10,7 +10,7 @@ __all__ = [
 
 from datetime import timedelta
 
-import pytz
+from dateutil import tz
 from zope.formlib.widget import CustomWidgetFactory
 
 from lp import _
@@ -56,7 +56,7 @@ class BaseSprintAttendanceAddView(LaunchpadFormView):
         # after the sprint. We will accept a time just before or just after
         # and map those to the beginning and end times, respectively, in
         # self.getDates().
-        time_zone = pytz.timezone(self.context.time_zone)
+        time_zone = tz.gettz(self.context.time_zone)
         from_date = self.context.time_starts.astimezone(time_zone)
         to_date = self.context.time_ends.astimezone(time_zone)
         self.starts_widget.from_date = from_date - timedelta(days=1)
@@ -142,16 +142,16 @@ class BaseSprintAttendanceAddView(LaunchpadFormView):
     @property
     def local_start(self):
         """The sprint start time, in the local time zone, as text."""
-        tz = pytz.timezone(self.context.time_zone)
-        return self.context.time_starts.astimezone(tz).strftime(
+        time_zone = tz.gettz(self.context.time_zone)
+        return self.context.time_starts.astimezone(time_zone).strftime(
             self._local_timeformat
         )
 
     @property
     def local_end(self):
         """The sprint end time, in the local time zone, as text."""
-        tz = pytz.timezone(self.context.time_zone)
-        return self.context.time_ends.astimezone(tz).strftime(
+        time_zone = tz.gettz(self.context.time_zone)
+        return self.context.time_ends.astimezone(time_zone).strftime(
             self._local_timeformat
         )
 
diff --git a/lib/lp/bugs/doc/bugnotification-email.rst b/lib/lp/bugs/doc/bugnotification-email.rst
index 9f07a1e..94a7a2b 100644
--- a/lib/lp/bugs/doc/bugnotification-email.rst
+++ b/lib/lp/bugs/doc/bugnotification-email.rst
@@ -585,12 +585,12 @@ method requires a from address, a to person, a body, a subject and a sending
 date for the mail.
 
     >>> from datetime import datetime
-    >>> import pytz
+    >>> from dateutil import tz
 
     >>> from_address = get_bugmail_from_address(lp_janitor, bug_four)
     >>> to_person = getUtility(IPersonSet).getByEmail("[e-mail]")
-    >>> sending_date = pytz.timezone("Europe/Prague").localize(
-    ...     datetime(2008, 5, 20, 11, 5, 47)
+    >>> sending_date = datetime(
+    ...     2008, 5, 20, 11, 5, 47, tzinfo=tz.gettz("Europe/Prague")
     ... )
 
     >>> notification_email = bug_four_notification_builder.build(
diff --git a/lib/lp/bugs/doc/externalbugtracker.rst b/lib/lp/bugs/doc/externalbugtracker.rst
index 84a1835..b2288ef 100644
--- a/lib/lp/bugs/doc/externalbugtracker.rst
+++ b/lib/lp/bugs/doc/externalbugtracker.rst
@@ -361,8 +361,8 @@ the time is.
 If the difference between what we and the remote system think the time
 is, an error is raised.
 
-    >>> import pytz
     >>> from datetime import datetime, timedelta, timezone
+    >>> from dateutil import tz
     >>> utc_now = datetime.now(timezone.utc)
     >>> class PositiveTimeSkewExternalBugTracker(TestExternalBugTracker):
     ...     def getCurrentDBTime(self):
@@ -417,7 +417,7 @@ than the UTC time.
 
     >>> class LocalTimeExternalBugTracker(TestExternalBugTracker):
     ...     def getCurrentDBTime(self):
-    ...         local_time = utc_now.astimezone(pytz.timezone("US/Eastern"))
+    ...         local_time = utc_now.astimezone(tz.gettz("US/Eastern"))
     ...         return local_time + timedelta(minutes=1)
     ...
     >>> bug_watch_updater.updateBugWatches(
diff --git a/lib/lp/bugs/tests/bugs-emailinterface.rst b/lib/lp/bugs/tests/bugs-emailinterface.rst
index b0d3575..5f87927 100644
--- a/lib/lp/bugs/tests/bugs-emailinterface.rst
+++ b/lib/lp/bugs/tests/bugs-emailinterface.rst
@@ -3193,7 +3193,7 @@ we'll create a new bug on firefox and link it to a remote bug.
     >>> no_priv = getUtility(IPersonSet).getByName("no-priv")
 
     >>> from datetime import datetime, timezone
-    >>> import pytz
+    >>> from dateutil import tz
     >>> creation_date = datetime(2008, 4, 12, 10, 12, 12, tzinfo=timezone.utc)
 
 We create the initial bug message separately from the bug itself so that
@@ -3238,7 +3238,7 @@ importing machinery.
     >>> bug_watch = getUtility(IBugWatchSet).get(bug_watch.id)
 
     >>> comment_date = datetime(
-    ...     2008, 5, 19, 16, 19, 12, tzinfo=pytz.timezone("Europe/Prague")
+    ...     2008, 5, 19, 16, 19, 12, tzinfo=tz.gettz("Europe/Prague")
     ... )
 
     >>> initial_mail = (
@@ -3265,7 +3265,7 @@ Now someone uses the email interface to respond to the comment that has
 been submitted.
 
     >>> comment_date = datetime(
-    ...     2008, 5, 20, 11, 24, 12, tzinfo=pytz.timezone("Europe/Prague")
+    ...     2008, 5, 20, 11, 24, 12, tzinfo=tz.gettz("Europe/Prague")
     ... )
 
     >>> reply_mail = (
@@ -3318,7 +3318,7 @@ to an email that isn't linked to the bug, the new message will be linked
 to the bug and will not have its bugwatch field set.
 
     >>> comment_date = datetime(
-    ...     2008, 5, 21, 11, 9, 12, tzinfo=pytz.timezone("Europe/Prague")
+    ...     2008, 5, 21, 11, 9, 12, tzinfo=tz.gettz("Europe/Prague")
     ... )
 
     >>> initial_mail = (
@@ -3338,7 +3338,7 @@ to the bug and will not have its bugwatch field set.
     >>> message = getUtility(IMessageSet).fromEmail(initial_mail, no_priv)
 
     >>> comment_date = datetime(
-    ...     2008, 5, 21, 12, 52, 12, tzinfo=pytz.timezone("Europe/Prague")
+    ...     2008, 5, 21, 12, 52, 12, tzinfo=tz.gettz("Europe/Prague")
     ... )
 
     >>> reply_mail = (
diff --git a/lib/lp/registry/browser/person.py b/lib/lp/registry/browser/person.py
index 1484dab..f6c7ca2 100644
--- a/lib/lp/registry/browser/person.py
+++ b/lib/lp/registry/browser/person.py
@@ -56,7 +56,7 @@ from operator import attrgetter, itemgetter
 from textwrap import dedent
 from urllib.parse import quote, urlencode
 
-import pytz
+from dateutil import tz
 from lazr.config import as_timedelta
 from lazr.delegates import delegate_to
 from lazr.restful.interface import copy_field
@@ -2043,9 +2043,7 @@ class PersonView(LaunchpadView, FeedsMixin, ContactViaWebLinksMixin):
     @property
     def time_zone_offset(self):
         """Return a string with offset from UTC"""
-        return datetime.now(pytz.timezone(self.context.time_zone)).strftime(
-            "%z"
-        )
+        return datetime.now(tz.gettz(self.context.time_zone)).strftime("%z")
 
 
 class PersonParticipationView(LaunchpadView):
@@ -4419,13 +4417,13 @@ class PersonEditTimeZoneView(LaunchpadFormView):
     @action(_("Update"), name="update")
     def action_update(self, action, data):
         """Set the time zone for the person."""
-        tz = data.get("time_zone")
-        if tz is None:
+        time_zone = data.get("time_zone")
+        if time_zone is None:
             raise UnexpectedFormData("No location received.")
         # XXX salgado, 2012-02-16, bug=933699: Use setLocation() because it's
         # the cheaper way to set the timezone of a person. Once the bug is
         # fixed we'll be able to get rid of this hack.
-        self.context.setLocation(None, None, tz, self.user)
+        self.context.setLocation(None, None, time_zone, self.user)
 
 
 def archive_to_person(archive):
diff --git a/lib/lp/services/webapp/doc/launchbag.rst b/lib/lp/services/webapp/doc/launchbag.rst
index 41e7b6e..091abaa 100644
--- a/lib/lp/services/webapp/doc/launchbag.rst
+++ b/lib/lp/services/webapp/doc/launchbag.rst
@@ -115,4 +115,4 @@ After the LaunchBag has been cleared, the correct time zone is returned.
     >>> launchbag.time_zone_name
     'Europe/Paris'
     >>> launchbag.time_zone
-    <... 'Europe/Paris' ...>
+    tzfile('.../Europe/Paris')
diff --git a/lib/lp/services/webapp/launchbag.py b/lib/lp/services/webapp/launchbag.py
index f20dfe0..34118a3 100644
--- a/lib/lp/services/webapp/launchbag.py
+++ b/lib/lp/services/webapp/launchbag.py
@@ -10,7 +10,7 @@ The collection of stuff we have traversed.
 import threading
 from datetime import timezone
 
-import pytz
+from dateutil import tz
 from zope.component import getUtility
 from zope.interface import implementer
 
@@ -161,7 +161,7 @@ class LaunchBag:
             if self.time_zone_name == "UTC":
                 self._store.time_zone = timezone.utc
             else:
-                self._store.time_zone = pytz.timezone(self.time_zone_name)
+                self._store.time_zone = tz.gettz(self.time_zone_name)
         return self._store.time_zone
 
 
diff --git a/lib/lp/services/worlddata/doc/vocabularies.rst b/lib/lp/services/worlddata/doc/vocabularies.rst
index 4e4ec6e..1f698b3 100644
--- a/lib/lp/services/worlddata/doc/vocabularies.rst
+++ b/lib/lp/services/worlddata/doc/vocabularies.rst
@@ -12,12 +12,10 @@ TimezoneName
 The TimezoneName vocabulary should only contain timezone names that
 do not raise an exception when instantiated.
 
-    >>> import pytz
+    >>> from dateutil import tz
     >>> timezone_vocabulary = vocabulary_registry.get(None, "TimezoneName")
     >>> for timezone in timezone_vocabulary:
-    ...     # Assign the return value of pytz.timezone() to the zone
-    ...     # variable to prevent printing out the return value.
-    ...     zone = pytz.timezone(timezone.value)
+    ...     _ = tz.gettz(timezone.value)
     ...
 
 LanguageVocabulary
diff --git a/lib/lp/services/worlddata/vocabularies.py b/lib/lp/services/worlddata/vocabularies.py
index bed76b1..72db756 100644
--- a/lib/lp/services/worlddata/vocabularies.py
+++ b/lib/lp/services/worlddata/vocabularies.py
@@ -7,8 +7,6 @@ __all__ = [
     "TimezoneNameVocabulary",
 ]
 
-import pytz
-import six
 from zope.component import getUtility
 from zope.interface import alsoProvides
 from zope.schema.vocabulary import SimpleTerm, SimpleVocabulary
@@ -19,14 +17,57 @@ from lp.services.worlddata.interfaces.timezone import ITimezoneNameVocabulary
 from lp.services.worlddata.model.country import Country
 from lp.services.worlddata.model.language import Language
 
-# create a sorted list of the common time zone names, with UTC at the start
-_values = sorted(six.ensure_text(tz) for tz in pytz.common_timezones)
-_values.remove("UTC")
-_values.insert(0, "UTC")
 
-_timezone_vocab = SimpleVocabulary.fromValues(_values)
+def _common_timezones():
+    """A list of useful, current time zone names.
+
+    This is inspired by `pytz.common_timezones`, which seems to be
+    approximately the list supported by `tzdata` with the additions of some
+    Canada- and US-specific names.  Since we're aiming for current rather
+    than historical zone names, `zone1970.tab` seems appropriate.
+    """
+    zones = set()
+    with open("/usr/share/zoneinfo/zone.tab") as zone_tab:
+        for line in zone_tab:
+            if line.startswith("#"):
+                continue
+            zones.add(line.rstrip("\n").split("\t")[2])
+    # Backward-compatible US zone names, still in common use.
+    zones.update(
+        {
+            "US/Alaska",
+            "US/Arizona",
+            "US/Central",
+            "US/Eastern",
+            "US/Hawaii",
+            "US/Mountain",
+            "US/Pacific",
+        }
+    )
+    # Backward-compatible Canadian zone names; see
+    # https://bugs.launchpad.net/pytz/+bug/506341.
+    zones.update(
+        {
+            "Canada/Atlantic",
+            "Canada/Central",
+            "Canada/Eastern",
+            "Canada/Mountain",
+            "Canada/Newfoundland",
+            "Canada/Pacific",
+        }
+    )
+    # pytz has this in addition to UTC.  Perhaps it's more understandable
+    # for people not steeped in time zone lore.
+    zones.add("GMT")
+
+    # UTC comes first, then everything else.
+    yield "UTC"
+    zones.discard("UTC")
+    yield from sorted(zones)
+
+
+_timezone_vocab = SimpleVocabulary.fromValues(_common_timezones())
 alsoProvides(_timezone_vocab, ITimezoneNameVocabulary)
-del _values
 
 
 def TimezoneNameVocabulary(context=None):
diff --git a/lib/lp/snappy/tests/test_snapbuildbehaviour.py b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
index 28ffc4b..6508601 100644
--- a/lib/lp/snappy/tests/test_snapbuildbehaviour.py
+++ b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
@@ -12,8 +12,8 @@ from textwrap import dedent
 from urllib.parse import urlsplit
 
 import fixtures
-import pytz
 from aptsources.sourceslist import SourceEntry
+from dateutil import tz
 from pymacaroons import Macaroon
 from testtools import ExpectedException
 from testtools.matchers import (
@@ -101,8 +101,8 @@ class FormatAsRfc3339TestCase(TestCase):
         self.assertEqual("2016-01-01T00:00:00Z", format_as_rfc3339(ts))
 
     def test_tzinfo_is_ignored(self):
-        tz = datetime(2016, 1, 1, tzinfo=pytz.timezone("US/Eastern"))
-        self.assertEqual("2016-01-01T00:00:00Z", format_as_rfc3339(tz))
+        time_zone = datetime(2016, 1, 1, tzinfo=tz.gettz("US/Eastern"))
+        self.assertEqual("2016-01-01T00:00:00Z", format_as_rfc3339(time_zone))
 
 
 class TestSnapBuildBehaviourBase(TestCaseWithFactory):
diff --git a/requirements/types.txt b/requirements/types.txt
index 221aad3..c42c102 100644
--- a/requirements/types.txt
+++ b/requirements/types.txt
@@ -4,7 +4,7 @@ types-beautifulsoup4==4.9.0
 types-bleach==3.3.1
 types-oauthlib==3.1.0
 types-psycopg2==2.9.21.4
-types-pytz==0.1.0
+types-python-dateutil==2.8.1
 types-requests==0.1.13
 types-six==0.1.9
 types-urllib3==1.26.25.4
diff --git a/setup.cfg b/setup.cfg
index ef37294..b2a0f29 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -79,12 +79,12 @@ install_requires =
     pymemcache
     pyparsing
     pystache
+    python-dateutil
     python-debian
     python-keystoneclient
     python-openid2
     python-subunit
     python-swiftclient
-    pytz
     PyYAML
     rabbitfixture
     requests