Merge ~mwhudson/debian-cd/+git/ubuntu:deb822-sources into ~ubuntu-cdimage/debian-cd/+git/ubuntu:main

[Michael Hudson-Doyle]

This is the partner to https://code.launchpad.net/~mwhudson/ubuntu-cdimage/+git/ubuntu-cdimage-1/+merge/457014, it uses the chdist-list apt setup from that branch to fetch packages from the pool rather than one locally created by debian-cd.

It has a branch set as a prerequisite that is a merge-up of my other outstanding (and somewhat more mundane) preparatory branches.

Unmerged commits

3d62c36... by Michael Hudson-Doyle on 2023-12-06

remove remaining uses of $MIRROR

d9442e7... by Michael Hudson-Doyle on 2023-12-05

remove references to ADIR and APTTMP

cc12bb7... by Michael Hudson-Doyle on 2023-12-05

use ubuntu-cdimage provided apt configuration for all apt work

diff --git a/CONF.sh b/CONF.sh
index 1b63e19..6cb382d 100644
--- a/CONF.sh
+++ b/CONF.sh
@@ -131,13 +131,12 @@ export ARCH=`dpkg --print-architecture`
 
 export FULLARCH=$ARCH${SUBARCH:++$SUBARCH}
 
-# IMPORTANT : The 4 following paths must be on the same partition/device.
+# IMPORTANT : The 2 following paths must be on the same partition/device.
 #	      If they aren't then you must set COPYLINK below to 1. This
 #	      takes a lot of extra room to create the sandbox for the ISO
 #	      images, however. Also, if you are using an NFS partition for
 #	      some part of this, you must use this option.
 # Paths to the mirrors
-export MIRROR=${MIRROR:-$CDIMAGE_ROOT/ftp}
 
 # Path of the temporary directory
 export TDIR=$CDIMAGE_ROOT/scratch/$PROJECT/$DIST/$IMAGE_TYPE/tmp
@@ -145,10 +144,6 @@ export TDIR=$CDIMAGE_ROOT/scratch/$PROJECT/$DIST/$IMAGE_TYPE/tmp
 # Path where the images will be written
 export OUT=$CDIMAGE_ROOT/scratch/$PROJECT/$DIST/$IMAGE_TYPE/debian-cd
 
-# Where we keep the temporary apt stuff.
-# This cannot reside on an NFS mount.
-export APTTMP=$CDIMAGE_ROOT/scratch/$PROJECT/$DIST/$IMAGE_TYPE/apt
-
 # Where extracted debootstrap scripts live
 export DEBOOTSTRAP=$CDIMAGE_ROOT/scratch/$PROJECT/$DIST/$IMAGE_TYPE/debootstrap
 
diff --git a/Makefile b/Makefile
index bfc52ff..35ce7d8 100644
--- a/Makefile
+++ b/Makefile
@@ -113,7 +113,6 @@ make_raw_img=$(BASEDIR)/tools/make-raw-img
 hardlink=$(BASEDIR)/tools/hardlink
 
 BDIR=$(TDIR)/$(CODENAME)-$(FULLARCH)
-ADIR=$(APTTMP)/$(CODENAME)-$(FULLARCH)
 SDIR=$(TDIR)/$(CODENAME)-src
 
 FIRSTDISKS=CD1 
@@ -215,9 +214,6 @@ endif
 ifndef BASEDIR
 	@echo BASEDIR undefined -- set up CONF.sh; false
 endif
-ifndef MIRROR
-	@echo MIRROR undefined -- set up CONF.sh; false
-endif
 ifndef ARCH
 	@echo ARCH undefined -- set up CONF.sh; false
 endif
@@ -236,7 +232,7 @@ endif
 ## INITIALIZATION ##
 
 # Creation of the directories needed
-init: ok $(OUT) $(TDIR) $(BDIR) $(SDIR) $(ADIR)
+init: ok $(OUT) $(TDIR) $(BDIR) $(SDIR)
 $(OUT):
 	$(Q)mkdir -p $(OUT)
 $(TDIR):
@@ -245,10 +241,6 @@ $(BDIR):
 	$(Q)mkdir -p $(BDIR)
 $(SDIR):
 	$(Q)mkdir -p $(SDIR)
-$(ADIR):
-	$(Q)mkdir -p $(ADIR)
-	$(Q)mkdir -p $(ADIR)/apt-ftparchive-db
-	$(Q)mkdir -p $(ADIR)/apt/preferences.d
 
 ## CLEANINGS ##
 
@@ -269,28 +261,13 @@ distclean: ok bin-distclean src-distclean
 bin-distclean:
 	$(Q)echo "Cleaning the binary build directory"
 	$(Q)rm -rf $(BDIR)
-	$(Q)rm -rf $(ADIR)
 src-distclean:
 	$(Q)echo "Cleaning the source build directory"
 	$(Q)rm -rf $(SDIR)
 
 ## APT ##
 
-# Configure apt stuff
-apt-setup: init $(ADIR)/apt-setup
-$(ADIR)/apt-setup:
-	@echo "Setting up archive keyring from host keyring"
-	$(Q)mkdir -p $(ADIR)/apt/trusted.gpg.d
-	$(Q)for file in /etc/apt/trusted.gpg.d/*; do \
-		if [ -e $$file ]; then \
-			ln -s $$file $(ADIR)/apt/trusted.gpg.d/$$(basename $$file); \
-		fi; \
-	done
-	# Create the apt state directory and update the database
-	$(Q)$(apt) update
-	touch $(ADIR)/apt-setup
-
-apt-update: apt-setup
+apt-update:
 	@echo "Apt-get is updating its files ..."
 	$(Q)$(apt) update
 
diff --git a/README b/README
index a447282..c75eebd 100644
--- a/README
+++ b/README
@@ -44,7 +44,6 @@ Edit the CONF.sh and change the PATHs for the mirror and so on.
 $ sensible-editor CONF.sh
 $ . CONF.sh
 $ make distclean
-$ make apt-setup
 $ make list COMPLETE=1 SIZELIMIT1=625000000 SRCSIZELIMIT=665000000
 $ make official_images
 [ alternatively, if you only want binary images:
@@ -91,10 +90,6 @@ The process of building a CD is composed of the following steps:
 
     $ make mirrorcheck
 
-- then we initialize the temporary directory used for the build:
-
-    $ make apt-setup
-
 - now you can decide what you want on your CDs
 
     $ make list TASK=tasks/debian-2.2 COMPLETE=1
diff --git a/build_all.sh b/build_all.sh
index 7e4b4ca..522a86a 100755
--- a/build_all.sh
+++ b/build_all.sh
@@ -24,13 +24,9 @@ do
 	CDREV=`git rev-parse HEAD`
 	echo "Using revision $CDREV of debian-cd"
 
-	if type find-mirror >/dev/null 2>&1; then
-		# TODO: nasty upcall to cdimage wrapper scripts
-		export MIRROR="$(find-mirror "$ARCH")"
-	fi
 	echo " ... cleaning"
 	make distclean
-	make apt-setup
+	make init
 
 	echo " ... selecting packages to include"
 	SIZE_ARGS=''
diff --git a/tools/apt-selection b/tools/apt-selection
index effb9bf..723db82 100755
--- a/tools/apt-selection
+++ b/tools/apt-selection
@@ -1,133 +1,19 @@
 #!/bin/sh
 
-# This is a little shell script that will launch apt-get in dry-run mode
-# to find all the dependencies of a specific package
+set -e
 
-# There's not set -e here because this script may fail !
-# Apt doesn't always work ... 
-# set -e
-
-# Get the configuration information if necessary
-if [ -z "$CODENAME" -o -z "$ARCH" -o -z "$APTTMP" ]; then
-	if [ -e CONF.sh ]; then
-		source CONF.sh
-	else
-		echo "Please set the good environment variables before "
-		echo "launching this program ..."
-		echo "Current values are :"
-		echo "CODENAME=$CODENAME"
-		echo "ARCH=$ARCH"
-		echo "APTTMP=$APTTMP"
-	fi
-fi
-if [ -z "$FULLARCH" ]; then
-	FULLARCH="$ARCH"
-fi
-
-options=" -q -o Dir=$APTTMP/$CODENAME-$FULLARCH/ \
-	  -o Dir::State=$APTTMP/$CODENAME-$FULLARCH/apt-state/ \
-	  -o Dir::Cache=$APTTMP/$CODENAME-$FULLARCH/apt-cache/ \
-	  -o Dir::Etc=$APTTMP/$CODENAME-$FULLARCH/apt/ \
-	  -o APT::Cache::AllVersions=0 \
-	  -o APT::Architecture=$ARCH "
-
-if [ -f "$BASEDIR/data/$CODENAME/multiarch/$ARCH" ]; then
-    SECONDARY_ARCH="$(cat $BASEDIR/data/$CODENAME/multiarch/$ARCH)"
-    options="$options -o APT::Architectures=$SECONDARY_ARCH "
-fi
-
-sections=main
-if [ "${NONFREE:-0}" != "0" -o "${EXTRANONFREE:-0}" != "0" ]; then
-	sections="$sections non-free"
-fi
-if [ "${CONTRIB:-0}" != "0" ]; then
-	sections="$sections contrib"
-fi
-if [ "${RESTRICTED:-0}" != "0" ]; then
-	sections="$sections restricted"
-fi
-if [ "${UNIVERSE:-0}" != "0" ]; then
-	sections="$sections universe"
-fi
-if [ "${MULTIVERSE:-0}" != "0" ]; then
-	sections="$sections multiverse"
-fi
-
-# Check for the necessary dirs and files ...
-if [ ! -d "$APTTMP/$CODENAME-$FULLARCH/apt-state/lists/partial" ]; then
-	mkdir -p "$APTTMP/$CODENAME-$FULLARCH/apt-state/lists/partial"
-fi
-if [ ! -d "$APTTMP/$CODENAME-$FULLARCH/apt-cache/archives/partial" ]; then
-	mkdir -p "$APTTMP/$CODENAME-$FULLARCH/apt-cache/archives/partial"
-fi
-if [ ! -d "$APTTMP/$CODENAME-$FULLARCH/apt" ]; then
-	mkdir -p "$APTTMP/$CODENAME-$FULLARCH/apt"
-fi
-
-case $MIRROR in
-	*:*)
-		SCHEMA=
-		;;
-	*)
-		SCHEMA=file:
-		;;
-esac
-
-if [ ! -e "$APTTMP/$CODENAME-$FULLARCH/apt/sources.list" ]; then
-	# Generating a correct sources.list file
-	echo "deb $SCHEMA$MIRROR $CODENAME $sections" \
-	> $APTTMP/$CODENAME-$FULLARCH/apt/sources.list
-	echo "deb-src $SCHEMA$MIRROR $CODENAME $sections" \
-	>> $APTTMP/$CODENAME-$FULLARCH/apt/sources.list
-	# Post-release updates
-	if [ "${UPDATES:-0}" != "0" ]; then
-	  echo "deb $SCHEMA$MIRROR $CODENAME-updates $sections" \
-	  >> $APTTMP/$CODENAME-$FULLARCH/apt/sources.list
-	  echo "deb-src $SCHEMA$MIRROR $CODENAME-updates $sections" \
-	  >> $APTTMP/$CODENAME-$FULLARCH/apt/sources.list
-	fi
-	if [ "${PROPOSED:-0}" != "0" ]; then
-	  echo "deb $SCHEMA$MIRROR $CODENAME-proposed $sections" \
-	  >> $APTTMP/$CODENAME-$FULLARCH/apt/sources.list
-	  echo "deb-src $SCHEMA$MIRROR $CODENAME-proposed $sections" \
-	  >> $APTTMP/$CODENAME-$FULLARCH/apt/sources.list
-	fi
-fi
-
-temp=$APTTMP/$CODENAME-$FULLARCH/temp.apt-selection
+export APT_CONFIG=$(eval "echo \$APT_CONFIG_${ARCH}")
 
 # Launch the command
 if [ "$1" = "update" -o "$1" = "check" -o "$1" = "download" ]; then
-	apt-get $options $@
-	exit $?
-elif [ "$1" = "cache" ]; then
-	shift
-	apt-cache $options $@
-	exit $?
+	apt-get $@
 elif [ "$1" = "config" ]; then
 	shift
-	apt-config $options $@
-	exit $?
-elif [ "$1" = "deselected" ]; then
-	shift
-	apt-get $options -s $@ > $temp
-	num=$?
-	#if [ $num -ne 0 ]; then 
-	    #echo ": Param: apt-selection deselected $@" >&2; 
-	    #exit $num;  
-	#fi
-	perl -ne 'print "$1\n" if /^Remv (\S+).*/' $temp | sort
-elif [ "$1" = "selected" ]; then
+	apt-config $@
+elif [ "$1" = "cache" ]; then
 	shift
-	apt-get $options -s $@ > $temp 
-	num=$?
-	#if [ $num -ne 0 ]; then 
-	#    echo "ERROR: Param: apt-selection selected $@" >&2; 
-	#    exit $num;  
-	#fi
-	perl -ne 'print "$1\n" if /^Inst (\S+).*/' $temp | sort
+	apt-cache -o APT::Cache::AllVersions=0 $@
 else
-	apt-get $options -s $@
-	exit $?
+	apt-get -s $@
 fi
 
diff --git a/tools/guess-mirror b/tools/guess-mirror
new file mode 100755
index 0000000..527d3c5
--- /dev/null
+++ b/tools/guess-mirror
@@ -0,0 +1,36 @@
+#!/usr/bin/python3
+
+# This script tries to work out the "main" mirror for where we are
+# pulling packages from the pool from -- by looking for the first
+# .sources file and taking the first URI from that.
+#
+# This is used by scanpackages and scansources to find the overrides
+# files when building the pool, so it's a little bit important that
+# this works for production builds but not very important if it
+# returns something bogus when doing something peculiar like building
+# an image out of the rebuild archive.
+
+import glob
+import os
+import subprocess
+import sys
+
+from debian import deb822
+
+
+def getconfig(v):
+    env = dict(os.environ, APT_CONFIG=os.environ["APT_CONFIG_" + os.environ["ARCH"]])
+    cmd = ["sh", "-c", "eval $(apt-config shell X {}); echo $X".format(v)]
+    cp = subprocess.run(cmd, env=env, encoding='utf-8', capture_output=True)
+    return cp.stdout.strip()
+
+
+sources_list_d = getconfig("Dir::Etc::sourceparts/d")
+
+first_sources_path = glob.glob(os.path.join(sources_list_d, '*.sources'))[0]
+
+with open(first_sources_path) as sources:
+    for para in deb822.Deb822.iter_paragraphs(sources):
+        if 'URIs' in para:
+            print(para['URIs'].split()[0])
+            sys.exit()
diff --git a/tools/list2cds b/tools/list2cds
index 1100519..e7884ee 100755
--- a/tools/list2cds
+++ b/tools/list2cds
@@ -25,7 +25,6 @@ my $maxcds = 1;
 my $overflowbincds = $ENV{'OVERFLOWBINCDS'} || 1;
 
 my $apt = "$ENV{'BASEDIR'}/tools/apt-selection";
-my $adir = "$ENV{'APTTMP'}/$ENV{'CODENAME'}-$ENV{'FULLARCH'}";
 my $dir = "$ENV{'TDIR'}/$ENV{'CODENAME'}-$ENV{'FULLARCH'}";
 my $verbose = $ENV{'VERBOSE'} || 0;
 
diff --git a/tools/list2src b/tools/list2src
index 098a527..2741a55 100755
--- a/tools/list2src
+++ b/tools/list2src
@@ -32,7 +32,6 @@ my $apt = "$ENV{'BASEDIR'}/tools/apt-selection";
 
 my $apt_lists_cmd = `$apt config shell LISTS Dir::State::lists/d`;
 my $apt_lists = `eval $apt_lists_cmd; echo \$LISTS`;
-my $mirror = $ENV{'MIRROR'};
 my $codename = $ENV{'CODENAME'};
 
 $| = 1;
@@ -72,10 +71,6 @@ foreach my $dist ( @distributions ) {
    }
 }
 
-if ($nonfree or $extranonfree) {
-	push @SOURCES, "$mirror/dists/$codename/non-free/source/Sources.gz";
-}
-
 foreach (@SOURCES) {
 	die "File `$_' cannot be found ..." if not -f;
 }
diff --git a/tools/scanpackages b/tools/scanpackages
index 5712559..2841fee 100755
--- a/tools/scanpackages
+++ b/tools/scanpackages
@@ -135,6 +135,8 @@ Tree "dists/$CODENAME" {
 }
 EOF
 
+MIRROR=$($BASEDIR/tools/guess-mirror)
+
 # Creating the indices directory
 if [ ! -d "$BDIR/indices" ]; then
     mkdir $BDIR/indices
diff --git a/tools/scansources b/tools/scansources
index f0fc703..b1057c9 100755
--- a/tools/scansources
+++ b/tools/scansources
@@ -97,6 +97,8 @@ Tree "dists/$CODENAME" {
 }
 EOF
 
+MIRROR=$($BASEDIR/tools/guess-mirror)
+
 # Creating the indices directory
 if [ ! -d "$SDIR/indices" ]; then
     mkdir $SDIR/indices