Merge lp:~cjwatson/txpkgupload/ipv6-ftp into lp:~lazr-developers/txpkgupload/trunk

[Colin Watson]

Implement FTP extensions for IPv6.

This is a backport from https://github.com/twisted/twisted/pull/1149; if and when it's merged into a released version of Twisted then we should drop this rather hacky backport and use that instead.

Portions of this are by William Grant, but I did some more work to get things like EPSV ALL working. Getting the tests backported here as well was unfortunately unreasonably difficult, but I've at least tested it manually and hopefully the fact that the Twisted PR has reasonable test coverage will be good enough.

[lp:~cjwatson/txpkgupload/ipv6-ftp]

44. By Colin Watson on 2019-05-31

Implement FTP extensions for IPv6.

This is a backport from https://github.com/twisted/twisted/pull/1149; if
and when it's merged into a released version of Twisted then we should
drop this rather hacky backport and use that instead.

Portions of this are by William Grant, but I did some more work to get
things like EPSV ALL working. Getting the tests backported here as well
was unfortunately unreasonably difficult, but I've at least tested it
manually and hopefully the fact that the Twisted PR has reasonable test
coverage will be good enough.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'src/txpkgupload/twistedftp.py'
--- src/txpkgupload/twistedftp.py	2018-07-04 10:04:49 +0000
+++ src/txpkgupload/twistedftp.py	2019-05-31 16:40:35 +0000
@@ -10,6 +10,7 @@
     ]
 
 import os
+import re
 import tempfile
 
 from twisted.application import (
@@ -24,7 +25,11 @@
     IRealm,
     Portal,
     )
-from twisted.internet import defer
+from twisted.internet import (
+    defer,
+    error,
+    reactor,
+    )
 from twisted.protocols import ftp
 from twisted.python import filepath
 from zope.interface import implementer
@@ -132,6 +137,182 @@
             "Only IFTPShell interface is supported by this realm")
 
 
+_AFNUM_IP = 1
+_AFNUM_IP6 = 2
+
+
+class UnsupportedNetworkProtocolError(Exception):
+    """Raised when the client requests an unsupported network protocol."""
+
+
+def decodeExtendedAddress(address):
+    """
+    Decode an FTP protocol/address/port combination, using the syntax
+    defined in RFC 2428 section 2.
+
+    @return: a 3-tuple of (protocol, host, port).
+    """
+    delim = address[0]
+    protocol, host, port, _ = address[1:].split(delim)
+    return protocol, host, int(port)
+
+
+def decodeExtendedAddressLine(line):
+    """
+    Decode an FTP response specifying a protocol/address/port combination,
+    using the syntax defined in RFC 2428 sections 2 and 3.
+
+    @return: a 3-tuple of (protocol, host, port).
+    """
+    match = re.search(r'\((.*)\)', line)
+    if match:
+        return decodeExtendedAddress(match.group(1))
+    else:
+        raise ValueError('No extended address found in "%s"' % line)
+
+
+class FTPWithEPSV(ftp.FTP):
+
+    epsvAll = False
+    supportedNetworkProtocols = (_AFNUM_IP, _AFNUM_IP6)
+
+    def connectionLost(self, reason):
+        ftp.FTP.connectionLost(self, reason)
+        self.epsvAll = False
+
+    def getDTPPort(self, factory, interface=''):
+        """
+        Return a port for passive access, using C{self.passivePortRange}
+        attribute.
+        """
+        for portn in self.passivePortRange:
+            try:
+                dtpPort = self.listenFactory(portn, factory,
+                                             interface=interface)
+            except error.CannotListenError:
+                continue
+            else:
+                return dtpPort
+        raise error.CannotListenError('', portn,
+            "No port available in range %s" %
+            (self.passivePortRange,))
+
+    def ftp_PASV(self):
+        if self.epsvAll:
+            return defer.fail(ftp.BadCmdSequenceError(
+                'may not send PASV after EPSV ALL'))
+        return ftp.FTP.ftp_PASV(self)
+
+    def _validateNetworkProtocol(self, protocol):
+        """
+        Validate the network protocol requested in an EPRT or EPSV command.
+
+        For now we just hardcode the protocols we support, since this layer
+        doesn't have a good way to discover that.
+
+        @param protocol: An address family number.  See RFC 2428 section 2.
+        @type protocol: L{str}
+
+        @raise FTPCmdError: If validation fails.
+        """
+        # We can't actually honour an explicit network protocol request
+        # (violating a SHOULD in RFC 2428 section 3), but let's at least
+        # validate it.
+        try:
+            protocol = int(protocol)
+        except ValueError:
+            raise ftp.CmdArgSyntaxError(protocol)
+        if protocol not in self.supportedNetworkProtocols:
+            raise UnsupportedNetworkProtocolError(
+                ','.join(str(p) for p in self.supportedNetworkProtocols))
+
+    def ftp_EPSV(self, protocol=''):
+        if protocol == 'ALL':
+            self.epsvAll = True
+            self.sendLine('200 EPSV ALL OK')
+            return defer.succeed(None)
+        elif protocol:
+            try:
+                self._validateNetworkProtocol(protocol)
+            except ftp.FTPCmdError:
+                return defer.fail()
+            except UnsupportedNetworkProtocolError as e:
+                self.sendLine(
+                    '522 Network protocol not supported, use (%s)' % e.args)
+                return defer.succeed(None)
+
+        # if we have a DTP port set up, lose it.
+        if self.dtpFactory is not None:
+            # cleanupDTP sets dtpFactory to none.  Later we'll do
+            # cleanup here or something.
+            self.cleanupDTP()
+        self.dtpFactory = ftp.DTPFactory(pi=self)
+        self.dtpFactory.setTimeout(self.dtpTimeout)
+        if not protocol or protocol == _AFNUM_IP6:
+            interface = '::'
+        else:
+            interface = ''
+        self.dtpPort = self.getDTPPort(self.dtpFactory, interface=interface)
+
+        port = self.dtpPort.getHost().port
+        self.reply(ftp.ENTERING_EPSV_MODE, port)
+        return self.dtpFactory.deferred.addCallback(lambda ign: None)
+
+    def ftp_PORT(self):
+        if self.epsvAll:
+            return defer.fail(ftp.BadCmdSequenceError(
+                'may not send PORT after EPSV ALL'))
+        return ftp.FTP.ftp_PORT(self)
+
+    def ftp_EPRT(self, extendedAddress):
+        """
+        Extended request for a data connection.
+
+        As described by U{RFC 2428 section
+        2<https://tools.ietf.org/html/rfc2428#section-2>}::
+
+            The EPRT command allows for the specification of an extended
+            address for the data connection.  The extended address MUST
+            consist of the network protocol as well as the network and
+            transport addresses.
+        """
+        if self.epsvAll:
+            return defer.fail(ftp.BadCmdSequenceError(
+                'may not send EPRT after EPSV ALL'))
+
+        try:
+            protocol, ip, port = decodeExtendedAddress(extendedAddress)
+        except ValueError:
+            return defer.fail(ftp.CmdArgSyntaxError(extendedAddress))
+        if protocol:
+            try:
+                self._validateNetworkProtocol(protocol)
+            except ftp.FTPCmdError:
+                return defer.fail()
+            except UnsupportedNetworkProtocolError as e:
+                self.sendLine(
+                    '522 Network protocol not supported, use (%s)' % e.args)
+                return defer.succeed(None)
+
+        # if we have a DTP port set up, lose it.
+        if self.dtpFactory is not None:
+            self.cleanupDTP()
+
+        self.dtpFactory = ftp.DTPFactory(
+            pi=self, peerHost=self.transport.getPeer().host)
+        self.dtpFactory.setTimeout(self.dtpTimeout)
+        self.dtpPort = reactor.connectTCP(ip, port, self.dtpFactory)
+
+        def connected(ignored):
+            return ftp.ENTERING_PORT_MODE
+
+        def connFailed(err):
+            err.trap(ftp.PortConnectionError)
+            return ftp.CANT_OPEN_DATA_CNX
+
+        return self.dtpFactory.deferred.addCallbacks(connected, connFailed)
+
+
 class FTPServiceFactory(service.Service):
     """A factory that makes an `FTPService`"""
 
@@ -142,7 +323,7 @@
         factory = ftp.FTPFactory(portal)
 
         factory.tld = root
-        factory.protocol = ftp.FTP
+        factory.protocol = FTPWithEPSV
         factory.welcomeMessage = "Launchpad upload server"
         factory.timeOut = idle_timeout
 
@@ -151,6 +332,6 @@
 
     @staticmethod
     def makeFTPService(port, root, temp_dir, idle_timeout):
-        strport = "tcp:%s" % port
+        strport = "tcp6:%s" % port
         factory = FTPServiceFactory(port, root, temp_dir, idle_timeout)
         return strports.service(strport, factory.ftpfactory)