Merge ~juliank/britney/+git/britney2-ubuntu:no-remove-pending-tests into ~juliank/britney/+git/britney2-ubuntu:master

[Julian Andres Klode]

467dfe8... by Julian Andres Klode on 2024-03-25

Use all-proposed in noble for all architectures

cdc015e... by Julian Andres Klode on 2024-03-25

Do not set all-proposed with migration-reference/0

b2436f7... by Julian Andres Klode on 2024-03-25

Only remove tests from pending for non-ignored results

This avoids endlessly requeuing the test if the test produces
an older result.

This will make tests "disappear" if the infrastructure returns
old results for newer triggers but avoids the problem right
now where we end up queuing the same tests every run.

Unmerged commits

b2436f7... by Julian Andres Klode on 2024-03-25

Only remove tests from pending for non-ignored results

This avoids endlessly requeuing the test if the test produces
an older result.

This will make tests "disappear" if the infrastructure returns
old results for newer triggers but avoids the problem right
now where we end up queuing the same tests every run.

cdc015e... by Julian Andres Klode on 2024-03-25

Do not set all-proposed with migration-reference/0

467dfe8... by Julian Andres Klode on 2024-03-25

Use all-proposed in noble for all architectures

a3da577... by Steve Langasek on 2024-03-22

Merge branch 'no-implicit-dependency-policy'

ad797bd... by Julian Andres Klode on 2024-03-19

Revert "Temporarily disable use of sqlite db for autopkgtests"

The database has been regenerated, so let's switch back to using
it to speed up britney.

This reverts commit e0fb5d881654438dda09143e6b48c6d153975232.

57710ee... by Michael Hudson-Doyle on 2024-03-19

autopkgtest: Fix the date creation and comparison for time_t hack

50d59b6... by Brian Murray on 2024-03-18

Fix a pep8 issue

bc3a622... by Julian Andres Klode on 2024-03-18

Temporarily queue noble armhf tests with all-proposed=1

This is needed for the time_t transition.

e5a438f... by Steve Langasek on 2024-02-14

Add support in code for disabling ImplicitDependencyPolicy, and disable it

08a27a5... by Łukasz Zemczak on 2024-02-01

Attempt to get swift results working properly by using a workaround from security-britney.

diff --git a/britney.conf b/britney.conf
index e6b1d4e..b6c82ed 100644
--- a/britney.conf
+++ b/britney.conf
@@ -57,7 +57,6 @@ BOUNTY_MIN_AGE    = 2
 HINTSDIR = data/%(SERIES)-proposed/Hints
 
 # hint permissions
-HINTS_ADCONRAD     = ALL
 HINTS_LANEY        = ALL
 HINTS_STEFANOR     = ALL
 HINTS_STGRABER     = ALL
@@ -82,7 +81,7 @@ HINTS_FREEZE       = block block-all
 #
 # naming a non-existent section will effectively disable new smooth
 # updates but still allow removals to occur
-SMOOTH_UPDATES    = badgers
+SMOOTH_UPDATES    = libs oldlibs
 
 IGNORE_CRUFT      = 0
 
@@ -90,6 +89,8 @@ REMOVE_OBSOLETE   = no
 
 CHECK_BUILDD      = no
 
+IMPLICIT_DEPS     = no
+
 ADT_ENABLE        = yes
 ADT_ARCHES        = amd64 i386 armhf ppc64el arm64
 ADT_AMQP          = amqp://test_request:[e-mail]
@@ -101,7 +102,7 @@ ADT_PPAS          =
 ADT_SHARED_RESULTS_CACHE =
 # Swift base URL with the results (must be publicly readable and browsable)
 # or file location if results are pre-fetched
-ADT_SWIFT_URL     = https://autopkgtest.ubuntu.com/results/
+ADT_SWIFT_URL     = https://objectstorage.prodstack5.canonical.com/swift/v1/AUTH_0f9aae918d5b4744bf7b827671c86842/
 # Base URL for autopkgtest site, used for links in the excuses
 ADT_CI_URL        = https://autopkgtest.ubuntu.com/
 # URL for the autopkgtest database, if used
diff --git a/britney.py b/britney.py
index b5deff2..8d8f427 100755
--- a/britney.py
+++ b/britney.py
@@ -236,6 +236,8 @@ from britney2.utils import (log_and_format_old_libraries,
 __author__ = 'Fabio Tranchitella and the Debian Release Team'
 __version__ = '2.0'
 
+# "temporarily" raise recursion limit for the auto hinter
+sys.setrecursionlimit(2000)
 
 class Britney(object):
     """Britney, the Debian testing updater script
@@ -521,8 +523,9 @@ class Britney(object):
         self._policy_engine.add_policy(RCBugPolicy(self.options, self.suite_info))
         if getattr(self.options, 'piuparts_enable', 'yes') == 'yes':
             self._policy_engine.add_policy(PiupartsPolicy(self.options, self.suite_info))
-        if getattr(self.options, 'adt_enable') == 'yes':
-            self._policy_engine.add_policy(AutopkgtestPolicy(self.options, self.suite_info))
+        add_autopkgtest_policy = getattr(self.options, 'adt_enable', 'no')
+        if add_autopkgtest_policy in ('yes', 'dry-run'):
+            self._policy_engine.add_policy(AutopkgtestPolicy(self.options, self.suite_info, dry_run=add_autopkgtest_policy == 'dry-run'))
         self._policy_engine.add_policy(AgePolicy(self.options, self.suite_info, MINDAYS))
         # XXX this policy results in asymmetric enforcement of
         # build-dependencies in the release pocket (nothing blocks
@@ -535,7 +538,13 @@ class Britney(object):
         self._policy_engine.add_policy(BlockPolicy(self.options, self.suite_info))
         # XXX re-enable once https://bugs.launchpad.net/launchpad/+bug/1868558 is fixed
         # self._policy_engine.add_policy(BuiltUsingPolicy(self.options, self.suite_info))
-        self._policy_engine.add_policy(ImplicitDependencyPolicy(self.options, self.suite_info))
+        # This policy is objectively terrible, it reduces britney runtime
+        # at the cost of giving only partial information about blocking
+        # packages during large transitions instead of giving full detail in
+        # update_output that the team can work through in parallel, thus
+        # dragging out complicated transitions. vorlon 20240214
+        if getattr(self.options, 'implicit_deps', 'yes') == 'yes':
+            self._policy_engine.add_policy(ImplicitDependencyPolicy(self.options, self.suite_info))
         if getattr(self.options, 'check_buildd', 'no') == 'yes':
             self._policy_engine.add_policy(BuiltOnBuilddPolicy(self.options, self.suite_info))
         self._policy_engine.add_policy(LPBlockBugPolicy(self.options, self.suite_info))
@@ -1461,6 +1470,9 @@ class Britney(object):
         """
         self.logger.info("> Processing hints from the auto hinter")
 
+        self.logger.info(">> Skipping auto hinting due to recursion problem")
+        return []
+
         sources_t = self.suite_info.target_suite.sources
         excuses = self.excuses
 
diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py
index 5faba7e..ac001c3 100644
--- a/britney2/policies/autopkgtest.py
+++ b/britney2/policies/autopkgtest.py
@@ -19,7 +19,7 @@
 import calendar
 import collections
 from copy import deepcopy
-from datetime import datetime
+from datetime import datetime, date
 from enum import Enum
 import os
 import json
@@ -117,7 +117,8 @@ class AutopkgtestPolicy(BasePolicy):
     reject the upload if any of those regress.
     """
 
-    def __init__(self, options, suite_info):
+    def __init__(self, options, suite_info, dry_run=False):
+
         super().__init__('autopkgtest', options, suite_info, {SuiteClass.PRIMARY_SOURCE_SUITE})
         # tests requested in this and previous runs
         # trigger -> src -> [arch]
@@ -126,6 +127,7 @@ class AutopkgtestPolicy(BasePolicy):
         self.testsuite_triggers = {}
         self.result_in_baseline_cache = collections.defaultdict(dict)
         self.database_path = os.path.join(self.state_dir, 'autopkgtest.db')
+        self.dry_run = dry_run
 
         # results map: trigger -> src -> arch -> [passed, version, run_id, seen]
         # - trigger is "source/version" of an unstable package that triggered
@@ -292,7 +294,7 @@ class AutopkgtestPolicy(BasePolicy):
         # Initialize AMQP connection
         self.amqp_channel = None
         self.amqp_file = None
-        if self.options.dry_run:
+        if self.options.dry_run or self.dry_run:
             return
 
         amqp_url = self.options.adt_amqp
@@ -475,7 +477,7 @@ class AutopkgtestPolicy(BasePolicy):
                         history_url = cloud_url % {
                             'h': srchash(testsrc), 's': testsrc,
                             'r': self.options.series, 'a': arch}
-                    if status == 'REGRESSION':
+                    if status in ['REGRESSION', 'RUNNING-REFERENCE']:
                         if self.options.adt_retry_url_mech == 'run_id':
                             retry_url = self.options.adt_ci_url + 'api/v1/retry/' + run_id
                         else:
@@ -892,7 +894,8 @@ class AutopkgtestPolicy(BasePolicy):
 
         # request new results from swift
         url = os.path.join(swift_url, self.swift_container)
-        url += '?' + urllib.parse.urlencode(query)
+        # XXX: Workaround for PS5 swift deployment - container name needs to be suffixed with /
+        url += '/?' + urllib.parse.urlencode(query)
         f = None
         try:
             f = self.download_retry(url)
@@ -1029,7 +1032,7 @@ class AutopkgtestPolicy(BasePolicy):
                                '       r.run_id FROM test AS t '
                                'LEFT JOIN result AS r ON t.id=r.test_id '
                                'WHERE t.release=? AND t.arch=? '
-                               'AND t.package=? AND r.run_id >= ?',
+                               'AND t.package=? AND r.run_id > ?',
                                (self.options.series, arch, src, latest_run_id)):
             exitcode, ver, triggers, run_id = row
             if not ver:
@@ -1048,7 +1051,7 @@ class AutopkgtestPolicy(BasePolicy):
 
             # parse recorded triggers in test result
             if triggers:
-                result_triggers = [i for i in triggers.split(' ', 1) if '/' in i]
+                result_triggers = [i for i in triggers.split(' ') if '/' in i]
             else:
                 self.logger.error('%s result has no ADT_TEST_TRIGGERS, ignoring')
                 continue
@@ -1068,13 +1071,10 @@ class AutopkgtestPolicy(BasePolicy):
                 'Fetched test result for %s/%s/%s %s (triggers: %s): %s',
                 src, ver, arch, run_id, result_triggers, result.name.lower())
 
-            # remove matching test requests
-            for trigger in result_triggers:
-                self.remove_from_pending(trigger, src, arch)
-
-            # add this result
+            # add this result and remove pending test requests if valid
             for trigger in result_triggers:
-                self.add_trigger_to_results(trigger, src, ver, arch, run_id, seen, result)
+                if self.add_trigger_to_results(trigger, src, ver, arch, run_id, seen, result):
+                    self.remove_from_pending(trigger, src, arch)
 
     def remove_from_pending(self, trigger, src, arch):
         try:
@@ -1094,10 +1094,10 @@ class AutopkgtestPolicy(BasePolicy):
             (trigsrc, trigver) = trigger.split('/', 1)
         except ValueError:
             self.logger.info('Ignoring invalid test trigger %s', trigger)
-            return
+            return False
         if trigsrc == src and apt_pkg.version_compare(ver, trigver) < 0:
             self.logger.debug('test trigger %s, but run for older version %s, ignoring', trigger, ver)
-            return
+            return False
 
         result = self.test_results.setdefault(trigger, {}).setdefault(
             src, {}).setdefault(arch, [Result.FAIL, None, '', 0])
@@ -1111,13 +1111,15 @@ class AutopkgtestPolicy(BasePolicy):
             result[2] = run_id
             result[3] = seen
 
+        return True
+
     def send_test_request(self, src, arch, triggers, huge=False):
         '''Send out AMQP request for testing src/arch for triggers
 
         If huge is true, then the request will be put into the -huge instead of
         normal queue.
         '''
-        if self.options.dry_run:
+        if self.options.dry_run or self.dry_run:
             return
 
         params = {'triggers': triggers}
@@ -1128,6 +1130,11 @@ class AutopkgtestPolicy(BasePolicy):
             qname = 'debci-huge-%s-%s' % (self.options.series, arch)
         else:
             qname = 'debci-%s-%s' % (self.options.series, arch)
+        # FIXME: Remove after the time_t transition
+        if (self.options.series == "noble" and
+                "migration-reference/0" not in triggers and
+                datetime.utcnow().date() < date(2024, 4, 4)):
+            params["all-proposed"] = "1"
         params['submit-time'] = datetime.strftime(datetime.utcnow(), '%Y-%m-%d %H:%M:%S%z')
 
         if self.amqp_channel:
@@ -1168,11 +1175,12 @@ class AutopkgtestPolicy(BasePolicy):
         if has_result:
             result_state = result[0]
             version = result[1]
-            baseline = self.result_in_baseline(src, arch)
             if result_state in {Result.OLD_PASS, Result.OLD_FAIL, Result.OLD_NEUTRAL}:
                 pass
             elif result_state == Result.FAIL and \
-                    baseline[0] in {Result.PASS, Result.NEUTRAL, Result.OLD_PASS, Result.OLD_NEUTRAL} and \
+                    self.result_in_baseline(src, arch)[0] in \
+                    {Result.PASS, Result.NEUTRAL, Result.OLD_PASS,
+                     Result.OLD_NEUTRAL} and \
                     self.options.adt_retry_older_than and \
                     result[3] + int(self.options.adt_retry_older_than) * SECPERDAY < self._now:
                 # We might want to retry this failure, so continue
@@ -1233,7 +1241,21 @@ class AutopkgtestPolicy(BasePolicy):
         result_reference = [Result.NONE, None, '', 0]
         if self.options.adt_baseline == 'reference':
             try:
-                result_reference = self.test_results[REF_TRIG][src][arch]
+                try:
+                    result_reference = self.test_results[REF_TRIG][src][arch]
+                except KeyError:
+                    uses_swift = not self.options.adt_swift_url.startswith('file://')
+                    # Without swift or autopkgtest.db we don't expect new results
+                    if hasattr(self,'db'):
+                        self.logger.info('Checking for new results for %s/%s for trigger %s', src, arch, REF_TRIG)
+                        self.fetch_sqlite_results(src, arch)
+                    elif uses_swift:
+                        self.logger.info('Checking for new results for %s/%s for trigger %s', src, arch, REF_TRIG)
+                        self.fetch_swift_results(self.options.adt_swift_url, src, arch)
+
+                    # do we have one now?
+                    result_reference = self.test_results[REF_TRIG][src][arch]
+
                 self.logger.debug('Found result for src %s in reference: %s',
                                   src, result_reference[0].name)
             except KeyError:
@@ -1270,9 +1292,6 @@ class AutopkgtestPolicy(BasePolicy):
         binaries_info = target_suite.binaries[arch]
 
         # determine current test result status
-        baseline_result = self.result_in_baseline(src, arch)[0]
-
-        # determine current test result status
         until = self.find_max_lower_force_reset_test(src, ver, arch)
 
         # Special-case triggers from linux-meta*: we cannot compare results
@@ -1297,6 +1316,9 @@ class AutopkgtestPolicy(BasePolicy):
             run_id = r[2]
 
             if r[0] in {Result.FAIL, Result.OLD_FAIL}:
+                # determine current test result status
+                baseline_result = self.result_in_baseline(src, arch)[0]
+
                 if baseline_result == Result.FAIL:
                     result = 'ALWAYSFAIL'
                 elif baseline_result in {Result.NONE, Result.OLD_FAIL}:
@@ -1354,6 +1376,7 @@ class AutopkgtestPolicy(BasePolicy):
         except KeyError:
             # no result for src/arch; still running?
             if arch in self.pending_tests.get(trigger, {}).get(src, []):
+                baseline_result = self.result_in_baseline(src, arch)[0]
                 if baseline_result != Result.FAIL and not self.has_force_badtest(src, ver, arch):
                     result = 'RUNNING'
                 else:
@@ -1420,10 +1443,10 @@ class AutopkgtestPolicy(BasePolicy):
             self.reset_hints = self.hints.search('force-reset-test')
 
         for hint in self.reset_hints:
-            self.logger.info('Checking hints for %s/%s/%s: %s' % (src, ver, arch, str(hint)))
             for mi in hint.packages:
                 if mi.package != src:
                     continue
+                self.logger.info('Checking hints for %s/%s/%s: %s' % (src, ver, arch, str(hint)))
                 if (mi.architecture in ['source', arch] and
                         mi.version != 'all' and
                         apt_pkg.version_compare(mi.version, ver) >= 0):
diff --git a/britney2/policies/email.py b/britney2/policies/email.py
index f500a79..349bf91 100644
--- a/britney2/policies/email.py
+++ b/britney2/policies/email.py
@@ -28,7 +28,7 @@ BOTS = {
     USER + "katie",
 }
 
-MESSAGE = """From: Ubuntu Release Team <[e-mail]>
+MESSAGE = """From: Ubuntu Release Team <[e-mail]>
 To: {recipients}
 X-Proposed-Migration: notice
 Subject: [proposed-migration] {source_name} {version} stuck in {series}-proposed for {age} day{plural}.
@@ -45,7 +45,7 @@ http://people.canonical.com/~ubuntu-archive/proposed-migration/{series}/update_e
 
 https://wiki.ubuntu.com/ProposedMigration
 
-If you have any questions about this email, please ask them in #ubuntu-release channel on Freenode IRC.
+If you have any questions about this email, please ask them in #ubuntu-release channel on libera.chat.
 
 Regards, Ubuntu Release Team.
 """
@@ -287,7 +287,7 @@ class EmailPolicy(BasePolicy, Rest):
                         )
                     )
                     server = smtplib.SMTP(self.email_host)
-                    server.sendmail("[e-mail]", emails, msg)
+                    server.sendmail("[e-mail]", emails, msg)
                     server.quit()
                     # record the age at which the mail should have been sent
                     last_sent = last_due
diff --git a/britney2/policies/sourceppa.py b/britney2/policies/sourceppa.py
index ea01e76..bbef00d 100644
--- a/britney2/policies/sourceppa.py
+++ b/britney2/policies/sourceppa.py
@@ -15,6 +15,7 @@ from britney2.policies.policy import BasePolicy, PolicyVerdict
 LAUNCHPAD_URL = "https://api.launchpad.net/1.0/"
 PRIMARY = LAUNCHPAD_URL + "ubuntu/+archive/primary"
 INCLUDE = ["~bileto-ppa-service/", "~ci-train-ppa-service/"]
+EXCLUDE = ["~ci-train-ppa-service/+archive/ubuntu/4810", "~ci-train-ppa-service/+archive/ubuntu/4813", "~ci-train-ppa-service/+archive/ubuntu/4815", "~ci-train-ppa-service/+archive/ubuntu/4816"]
 
 
 class SourcePPAPolicy(BasePolicy, Rest):
@@ -105,6 +106,8 @@ class SourcePPAPolicy(BasePolicy, Rest):
         sourceppa = self.lp_get_source_ppa(source_name, version) or ""
         verdict = excuse.policy_verdict
         self.source_ppas_by_pkg[source_name][version] = sourceppa
+        if [team for team in EXCLUDE if team in sourceppa]:
+            return PolicyVerdict.PASS
         if not [team for team in INCLUDE if team in sourceppa]:
             return PolicyVerdict.PASS
 
diff --git a/britney2/policies/sruadtregression.py b/britney2/policies/sruadtregression.py
index 1c8ac82..6fa8985 100644
--- a/britney2/policies/sruadtregression.py
+++ b/britney2/policies/sruadtregression.py
@@ -171,7 +171,7 @@ class SRUADTRegressionPolicy(BasePolicy, Rest):
                     bug_mail = "[e-mail]" % bug
                     server = smtplib.SMTP(self.email_host)
                     server.sendmail(
-                        "[e-mail]",
+                        "[e-mail]",
                         bug_mail,
                         MESSAGE.format(**locals()),
                     )
diff --git a/britney_nobreakall.conf b/britney_nobreakall.conf
index 5da361b..4576790 100644
--- a/britney_nobreakall.conf
+++ b/britney_nobreakall.conf
@@ -57,7 +57,6 @@ BOUNTY_MIN_AGE    = 2
 HINTSDIR = data/%(SERIES)-proposed/Hints
 
 # hint permissions
-HINTS_ADCONRAD     = ALL
 HINTS_LANEY        = ALL
 HINTS_STEFANOR     = ALL
 HINTS_STGRABER     = ALL