Merge ~pjdc/ubuntu-mirror-charm/+git/ubuntu-mirror-charm:add-extras into ubuntu-mirror-charm:master

[Paul Collins]

add support for hosting extras.ubuntu.com

[Canonical IS Mergebot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Haw Loeung (hloeung)]

LGTM

review: Approve (+1)

[Canonical IS Mergebot (canonical-is-mergebot)]

Change successfully merged at revision f32f68414d12ccf92341c00762a5bf03243468bd

diff --git a/config.yaml b/config.yaml
index 582b256..b704dc4 100644
--- a/config.yaml
+++ b/config.yaml
@@ -271,6 +271,63 @@ options:
     description: >
       Local path of the key file to use when triggering downstream
       mirrors.  If empty, downstream mirrors are not triggered.
+  mirror_extras_name:
+    default: "extras.ubuntu.com"
+    type: string
+    description: "FQDN of the Ubuntu Extras mirror"
+  mirror_extras_aliases:
+    default: '["*.extras.ubuntu.com", "extras.ubuntulinux.org", "*.extras.ubuntulinux.org"]'
+    type: string
+    description: "List of Apache aliases for the Ubuntu Extras mirror"
+  mirror_extras_path:
+    default: "/srv/ftp.root/extras"
+    type: string
+    description: "Root location of mirrored files for the Ubuntu Extras mirror"
+  mirror_extras_description:
+    default: "Ubuntu Extras"
+    type: string
+    description: "A brief welcome message for the Ubuntu Extras mirror"
+  mirror_extras_rsync_log:
+    default: true
+    type: boolean
+    description: "Whether to log rsync requests for the Ubuntu Extras mirror"
+  mirror_extras_command:
+    default: "mirror-2stage.sh"
+    type: string
+    description: "The command to use to sync the Ubuntu Extras mirror"
+  mirror_extras_source_url:
+    default: "rsync://extras.ubuntu.com/extras"
+    type: string
+    description: "The URL the Ubuntu Extras mirror will be fetched from"
+  mirror_extras_rsync_auth:
+    default: '{}'
+    type: string
+    description: "Optional rsync authentication details for mirror_extras_source_url"
+  mirror_extras_rsync_module:
+    default: "extras"
+    type: string
+    description: "The name of the rsync module for this mirror role"
+  mirror_extras_sync_time:
+    default: "ondemand"
+    type: string
+    description: "When mirror updates should be run.  Either a cron(5) format time specification or 'ondemand' for ssh triggering"
+  mirror_extras_trigger:
+    default: ""
+    type: string
+    description: "A base64 string containing the ssh trigger public key"
+  mirror_extras_downstream_mirrors:
+    default: ""
+    type: "string"
+    description: >
+      A space-separated list of mirrors to trigger following a sync.
+      Username defaults to the mirror_user config setting, and
+      otherwise may be specified by "user@host" syntax.
+  mirror_extras_trigger_keyfile:
+    default: ""
+    type: "string"
+    description: >
+      Local path of the key file to use when triggering downstream
+      mirrors.  If empty, downstream mirrors are not triggered.
   mirror_simple-streams_name:
     default: "streams.canonical.com"
     type: string
diff --git a/files/check-mirror.sh b/files/check-mirror.sh
index 003b48a..7714ce4 100755
--- a/files/check-mirror.sh
+++ b/files/check-mirror.sh
@@ -80,6 +80,10 @@ case ${role} in
 	unsigned=ephemeral-v3/daily/focal/amd64/
 	signed=${unsigned}.gpg  # not used
 	;;
+    extras)
+        unsigned=ubuntu/dists/trusty/Release
+        signed=${unsigned}.gpg
+        ;;
     *)
         echo "Unknown mirror role, ${role}"
         exit 2
@@ -102,6 +106,9 @@ case ${role} in
     cloud-images)
         keyring_file=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg
         ;;
+    extras)
+	keyring_file=/usr/local/share/ubuntu-mirror-charm.gpg
+	;;
     linuxcontainers)
 	keyring_file=/usr/local/share/ubuntu-mirror-charm.gpg
 	;;
diff --git a/files/check-updates.sh b/files/check-updates.sh
index f1ebe86..3fb11bc 100755
--- a/files/check-updates.sh
+++ b/files/check-updates.sh
@@ -35,6 +35,10 @@ function days {
 }
 
 case $NAME in
+    extras.ubuntu.com)
+	# This will never be updated again.
+	crit_age=$(days 3650)
+	;;
     images.maas.io)
 	crit_age=$(days 7)
 	;;
diff --git a/hooks/Config.py b/hooks/Config.py
index 7b90858..a648bc2 100755
--- a/hooks/Config.py
+++ b/hooks/Config.py
@@ -239,6 +239,7 @@ class Config:
         return [
             "cdimage",
             "cloud-images",
+            "extras",
             "linuxcontainers",
             "maas-images",
             "old-releases",
diff --git a/keys/ubuntu-extras.asc b/keys/ubuntu-extras.asc
new file mode 100644
index 0000000..508b9e0
--- /dev/null
+++ b/keys/ubuntu-extras.asc
@@ -0,0 +1,17 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGiBEyXadoRBADTUoaVczNG3ras9/nqhHVduWDjxi0wbhMfRpciB2NK9T5YVVPq
+LPDtRCpso07ackIwzDalizzvXm6bgJVWrg5//F8r7k/OowWgJ6B+SRzUzvROIR4m
+zt/HnBhYU2otPY7QWcGMcOhjMnYCgKTn20aJfmr+8CcvM87XLhylUTwoWwCgtIQC
+a20BjPQHrArjeu+YxnJi1vsD/A0rmQdF633pPr0cxum1CCoQn+4kpOIN7qKAppik
+Rfn7Cl+Pk4WYABtd//W7VjDcdfLHJImRR/y5PZVjHQHpy4b2UQ+Yrb3EFZdVuQZB
+YKwLmAEyk7i9/QmSSF+pte3srXLEbt7bBgVXwwB6om/x9JpbIcwJcixDBVMNjMOt
+KIKJA/4uQEXws1mKjSNEopCeAhY6mTnpCh7L/eqCMhBfZrtjDlbw62+lahpgQmTI
+x74YlB7qOAw5yHBPnOJ2hlY98GPzd0VS9xucggwb8RMBjz2ePPm/4T9Z+ScOKGxn
+P/MQWUVoXB1drjECuKy54QRzhltZwPZJF3m5YGbvDasIBtrY37RCVWJ1bnR1IEV4
+dHJhcyBBcmNoaXZlIEF1dG9tYXRpYyBTaWduaW5nIEtleSA8ZnRwbWFzdGVyQHVi
+dW50dS5jb20+iGAEExECACAFAkyXadoCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIX
+gAAKCRAWEm06PlwRksuCAJ9LLS1BvhqMOKfq5QWMqCeZ1R0howCdFAuj5kOgPajo
+AcakFvDQqwgerPk=
+=iZIY
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/templates/apache-extras.tmpl b/templates/apache-extras.tmpl
new file mode 100644
index 0000000..469452f
--- /dev/null
+++ b/templates/apache-extras.tmpl
@@ -0,0 +1,44 @@
+#for $port in $ports
+#set $sockets = ' '.join(sorted(['{}:{}'.format(address, port) for address in $addresses]))
+<VirtualHost ${sockets}>
+    ServerName ${name}
+    #for $alias in $aliases
+    ServerAlias ${alias}
+    #end for
+
+    CustomLog ${logdir}/${name}-access.log combined-port
+    ErrorLog ${logdir}/${name}-error.log
+    LogFormat "%h %p %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined-port
+
+    DocumentRoot /srv/${name}/www
+    # Munge the Cache-Control/Expires response headers to try make
+    # transparent proxy servers DTRT.
+    <Files ~ "Release(\.gpg)?|InRelease|Packages\.(xz|bz2|gz)|Sources\.(xz|bz2|gz)?|Translation-.*?$">
+        ExpiresActive On
+        ExpiresDefault "modification plus 1800 seconds"
+        Header append Cache-Control "proxy-revalidate"
+    </Files>
+    # Allow intermediate caches to cache .deb|.udeb|.ddeb files for a day.
+    # This could be increased in the future since deb files are the perfect
+    # case for cacheable resources (invariant, versioned URLs, and external
+    # integrity verification)
+    <FilesMatch ".+\.[ud]?deb$">
+        Header append Cache-Control "max-age=86400"
+    </FilesMatch>
+    # Allow intermediate caches to store by-hash files for up to a day.
+    # Again, these files are by definition invariant so they are perfectly
+    # cacheable.
+    <FilesMatch "\/by-hash\/">
+        Header append Cache-Control "max-age=86400"
+    </FilesMatch>
+
+    FileETag MTime Size
+    <Directory "/srv/${name}/">
+        Require all granted
+        IndexOptions NameWidth=* +SuppressDescription
+        Options +Indexes +FollowSymLinks
+        IndexIgnore favicon.ico
+    </Directory>
+</VirtualHost>
+
+#end for
diff --git a/tests/unit/test_templates.py b/tests/unit/test_templates.py
index ea2366b..203a7b2 100644
--- a/tests/unit/test_templates.py
+++ b/tests/unit/test_templates.py
@@ -280,3 +280,18 @@ class TestOldReleases(TemplateTestCase, unittest.TestCase):
     _role = 'old-releases'
     _search_list = OLD_RELEASES_SEARCH_LIST
     _thirdparty_search_list = None  # There are probably not going to be third-party mirrors of this.
+
+
+EXTRAS_SEARCH_LIST = {
+    'addresses': ['*'],
+    'name': 'extras.ubuntu.com',
+    'aliases': ['*.extras.ubuntu.com', 'extras.ubuntulinux.org', '*.extras.ubuntulinux.org'],
+    'ports': [80],
+    'logdir': '/var/log/apache2',
+}
+
+
+class TestExtras(TemplateTestCase, unittest.TestCase):
+    _role = 'extras'
+    _search_list = EXTRAS_SEARCH_LIST
+    _thirdparty_search_list = None  # There are probably not going to be third-party mirrors of this.
diff --git a/tests/unit/testdata/extras/extras.txt b/tests/unit/testdata/extras/extras.txt
new file mode 100644
index 0000000..88690e5
--- /dev/null
+++ b/tests/unit/testdata/extras/extras.txt
@@ -0,0 +1,41 @@
+<VirtualHost *:80>
+    ServerName extras.ubuntu.com
+    ServerAlias *.extras.ubuntu.com
+    ServerAlias extras.ubuntulinux.org
+    ServerAlias *.extras.ubuntulinux.org
+
+    CustomLog /var/log/apache2/extras.ubuntu.com-access.log combined-port
+    ErrorLog /var/log/apache2/extras.ubuntu.com-error.log
+    LogFormat "%h %p %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined-port
+
+    DocumentRoot /srv/extras.ubuntu.com/www
+    # Munge the Cache-Control/Expires response headers to try make
+    # transparent proxy servers DTRT.
+    <Files ~ "Release(\.gpg)?|InRelease|Packages\.(xz|bz2|gz)|Sources\.(xz|bz2|gz)?|Translation-.*?$">
+        ExpiresActive On
+        ExpiresDefault "modification plus 1800 seconds"
+        Header append Cache-Control "proxy-revalidate"
+    </Files>
+    # Allow intermediate caches to cache .deb|.udeb|.ddeb files for a day.
+    # This could be increased in the future since deb files are the perfect
+    # case for cacheable resources (invariant, versioned URLs, and external
+    # integrity verification)
+    <FilesMatch ".+\.[ud]?deb$">
+        Header append Cache-Control "max-age=86400"
+    </FilesMatch>
+    # Allow intermediate caches to store by-hash files for up to a day.
+    # Again, these files are by definition invariant so they are perfectly
+    # cacheable.
+    <FilesMatch "\/by-hash\/">
+        Header append Cache-Control "max-age=86400"
+    </FilesMatch>
+
+    FileETag MTime Size
+    <Directory "/srv/extras.ubuntu.com/">
+        Require all granted
+        IndexOptions NameWidth=* +SuppressDescription
+        Options +Indexes +FollowSymLinks
+        IndexIgnore favicon.ico
+    </Directory>
+</VirtualHost>
+