Merge ~andersson123/britney:no-content-length-check-checksum-instead into ~ubuntu-release/britney/+git/britney2-ubuntu:master

[Tim Andersson]

[Brian Murray (brian-murray)]

review: Needs Fixing

[Tim Andersson (andersson123)]

amended, please re-review

Unmerged commits

d5c0c1f... by Tim Andersson on 2024-04-24

feat: check database checksum instead of content-length header

autopkgtest-cloud will now serve:
autopkgtest.ubuntu.com/static/autopkgtest.db.sha256

Britney now calculates the sha256 of the newly downloaded db locally and
checks that it matches the sha256 file served by autopkgtest-cloud,
instead of checking that the content-length header matches the
size of the new downloaded database.

Since the most recent apache2 security update in focal [1], the
content-length header isn't served by default, and it seems that when
it is served it's not entirely accurate. This check has become
brittle, and so we have implemented this new mechanism.

[1] https://bugs.launchpad.net/ubuntu/+source/apache2/+bug/2061816

diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py
index de7af7b..b8ee55a 100644
--- a/britney2/policies/autopkgtest.py
+++ b/britney2/policies/autopkgtest.py
@@ -30,6 +30,7 @@ import re
 import socket
 import sqlite3
 import sys
+import hashlib
 import time
 import urllib.parse
 from urllib.error import HTTPError
@@ -169,8 +170,10 @@ class AutopkgtestPolicy(BasePolicy):
 
     def fetch_db(self):
         f = None
+        local_db_sha = hashlib.sha256()
         try:
             f = self.download_retry(self.options.adt_db_url)
+            chksum = self.download_retry(self.options.adt_db_url + ".sha256").read().rstrip()
             http_code = f.getcode()
             # file:/// urls don't have the http niceties
             if not http_code or http_code == 200:
@@ -180,10 +183,10 @@ class AutopkgtestPolicy(BasePolicy):
                         data=f.read(2048*1024)
                         if not data:
                             break
+                        local_db_sha.update(data)
                         f_out.write(data)
-                content_length = f.getheader('content-length')
-                if http_code and content_length and os.path.getsize(new_file) != content_length:
-                    self.logger.info('Short read downloading autopkgtest results')
+                if http_code and local_db_sha.hexdigest() != chksum:
+                    self.logger.info("autopkgtest.db local checksum does not match downloaded checksum!")
                     os.unlink(new_file)
                 else:
                     os.rename(new_file, self.database_path)