Kernel 6.6.30, Python 3.12

- Adjustments to create the appliance on Debian
- use grub-install from appliance (not from build host)
- fixes

.gitignore

@@ -1,13 +1,13 @@
 appliances/*
 !appliances/base/
 !appliances/default/
-build/
+build
-kernel/
+kernel
-loop/
+loop
-packages/
+packages
-repos/
+repos
-distfiles/
+distfiles
-images/
+images
 *.cfg
 !appliances/default/default.cfg
 latest-stage3.txt

Makefile

@@ -66,7 +66,7 @@ portage_package_files = $(patsubst appliances/$(APPLIANCE)/package.%,$(CHROOT)/e
 default_make_conf = $(wildcard appliances/default/make.conf)
 appliance_make_conf = $(wildcard appliances/$(APPLIANCE)/make.conf)
 portage_make_conf_local = $(CHROOT)/etc/portage/make.conf.local
-appliance_profile := default/linux/amd64/17.1/systemd
+appliance_profile := default/linux/amd64/23.0/systemd
 
 # Allow appliance to override variables
 -include appliances/default/default.cfg
@@ -112,7 +112,11 @@ endif
 
 gcc_config = $(inroot) gcc-config 1
 
-export APPLIANCE ACCEPT_KEYWORDS CHROOT EMERGE HEADLESS M4 M4C inroot
+ifneq ($(shell grep 'CONFIG_MODULES=y' $(KERNEL_CONFIG)), )
+        KERNEL_MODULES_PREPARE = modules_prepare
+endif
+
+export APPLIANCE ACCEPT_KEYWORDS CHROOT EMERGE HEADLESS M4 M4C inroot gcc_config
 export HOSTNAME MAKEOPTS TIMEZONE USEPKG WORLD 
 export USEPKG RSYNC_MIRROR
 export EXTERNAL_KERNEL KERNEL_PKG KERNEL_PATH KERNEL_CONFIG
@@ -128,19 +132,19 @@ sync_portage: $(PORTAGE_DIR)
 
 $(PORTAGE_DIR):
 	@print Grabbing the portage tree
-	git clone --depth=1 git://github.com/gentoo/gentoo.git $(PORTAGE_DIR)
+	git clone --depth=1 https://github.com/gentoo/gentoo.git $(PORTAGE_DIR)
 
 $(CHROOT)/etc/portage/%: configs/portage/%
 	COPY --recursive $< /etc/portage/
 
-$(CHROOT)/etc/fstab: configs/fstab.in
+fstab:
-ifeq ($(VIRTIO),YES)
+	@print Create new /etc/fstab
-	$(M4) -DDRIVE=sda $< > $@
+	env echo -e "# <fs>\t\t<mountpoint>\t<type>\t<opts>\t<dump/pass>" > $(CHROOT)/etc/fstab
-else
+	env echo -e "LABEL=ROOT\t/\t\text4\tnoatime\t1 1" >> $(CHROOT)/etc/fstab
-	$(M4) -DDRIVE=vda $< > $@
+	env echo -e "/.swap\t\tnone\t\tswap\tsw\t0 0" >> $(CHROOT)/etc/fstab
-endif
+	env echo -e "#LABEL=SWAP\tnone\t\tswap\tsw\t0 0" >> $(CHROOT)/etc/fstab
 
-$(PREPROOT): $(STAGE3) $(PORTAGE_DIR) $(foreach repo,$(REPO_NAMES),$(REPO_DIR)/$(repo)) $(CHROOT)/etc/fstab $(etc_portage) $(portage_default_package_files) $(portage_package_files)
+$(PREPROOT): $(STAGE3) $(PORTAGE_DIR) $(foreach repo,$(REPO_NAMES),$(REPO_DIR)/$(repo)) fstab $(etc_portage) $(portage_default_package_files) $(portage_package_files)
 	mkdir -p $(VA_PKGDIR) $(DISTDIR)
 	@print Creating swap file: `basename $(SWAP_FILE)`
 	dd if=/dev/zero of=$(SWAP_FILE) bs=1M count=$(SWAP_SIZE)
@@ -225,7 +229,7 @@ ifneq ($(EXTERNAL_KERNEL),YES)
 		cp -a $(CHROOT)/usr/src/linux-*/* $(shell cat $(KERNEL_PATH)); \
 		RUN $(EMERGE) -C sys-kernel/$(KERNEL_PKG); \
 		cp $(KERNEL_CONFIG) $(shell cat $(KERNEL_PATH))/.config; \
-##		RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) oldconfig modules_prepare; \
+		RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) oldconfig $(KERNEL_MODULES_PREPARE); \
 	fi
 endif
 	touch $(KERNEL_SRC)
@@ -338,14 +342,16 @@ $(RAW_IMAGE): $(STAGE4_TARBALL) scripts/grub.shell scripts/motd.sh
 	parted -s $(RAW_IMAGE).tmp set 1 boot on
 	sync
 	losetup --show --find --partscan $(RAW_IMAGE).tmp > partitions
-	mkfs.ext4 -O sparse_super,^has_journal -L "$(APPLIANCE)"_root -m 0 `cat partitions`p1
+	mkfs.ext4 -O sparse_super,^has_journal -L ROOT -m 0 `cat partitions`p1
 	mkdir $(CHROOT)
 	mount -o noatime `cat partitions`p1 $(CHROOT)
 	tar -xf $(STAGE4_TARBALL) --numeric-owner $(COPY_ARGS) -C $(CHROOT)
 	motd.sh $(EXTERNAL_KERNEL) $(VIRTIO) $(DISK_SIZE) $(SWAP_SIZE) $(DASH) $(VA_ARCH) > $(CHROOT)/etc/motd
 ifneq ($(EXTERNAL_KERNEL),YES)
-	echo '(hd0) ' `cat partitions` > device-map
+	echo '(hd0) ' `cat partitions` > $(CHROOT)/device-map
-	$(CHROOT)/usr/sbin/grub-install --no-floppy --grub-mkdevicemap=device-map --directory=$(CHROOT)/usr/lib/grub/i386-pc --boot-directory=$(CHROOT)/boot `cat partitions`
+	cp partitions $(CHROOT)/partitions
+	RUN /usr/sbin/grub-install --no-floppy --grub-mkdevicemap=device-map --directory=/usr/lib/grub/i386-pc --boot-directory=/boot `cat partitions`
+	rm $(CHROOT)/device-map  $(CHROOT)/partitions
 endif
 	umount $(CHROOT)
 	rmdir $(CHROOT)
@@ -373,7 +379,7 @@ xva: $(XVA_IMAGE)
 $(VMDK_IMAGE): $(RAW_IMAGE)
 	@print Creating `basename $(VMDK_IMAGE)`
 ifeq ($(VMDK_TYPE),SCSI)
-	qemu-img convert -f raw -O vmdk -o adapter_type=lsilogic,subformat=streamOptimized,compat6 $(RAW_IMAGE) $(VMDK_IMAGE).tmp
+	qemu-img convert -f raw -O vmdk -o adapter_type=lsilogic,subformat=streamOptimized,hwversion=7 $(RAW_IMAGE) $(VMDK_IMAGE).tmp
 else
 	qemu-img convert -f raw -O vmdk $(RAW_IMAGE) $(VMDK_IMAGE).tmp
 endif
@@ -387,7 +393,7 @@ export NUM_CPUS MEM_SIZE
 
 $(OVA_IMAGE): $(VMDK_IMAGE)
 	@print Creating `basename $(OVA_IMAGE)`
-	cd $(IMAGES) && mkova.sh $(APPLIANCE) /usr/share/open-vmdk/template.ovf $(VMDK_IMAGE)
+	cd $(IMAGES) && mkova.sh $(APPLIANCE) /usr/share/open-vmdk/template-hw7.ovf $(VMDK_IMAGE)
 
 ova: $(OVA_IMAGE)
 
@@ -481,4 +487,4 @@ help:
 	@print 'Example to build the base appliance'
 	@echo 'make APPLIANCE=base HEADLESS=YES VIRTIO=YES stage4 qcow clean'
 
-.PHONY: qcow vmdk ova clean realclean distclean stage4 image stage4 help appliance-list eclean sync_portage sync_stage3 checksums
+.PHONY: qcow vmdk ova clean realclean distclean stage4 image stage4 help appliance-list eclean sync_portage sync_stage3 checksums fstab

appliances/default/Makefile

@@ -13,15 +13,16 @@ $(HARDENED):
 	RUN $(EMERGE) $(USEPKG) --oneshot binutils virtual/libc
 	RUN $(EMERGE) --depclean --with-bdeps=n
 	-$(gcc_config)
+	RUN /usr/bin/gcc --version
 	RUN $(EMERGE) $(USEPKG) --emptytree @world
 	RUN $(EMERGE) --depclean --with-bdeps=n
 	RUN bash -c 'yes YES | etc-update --automode -9'
-##ifneq ($(EXTERNAL_KERNEL),YES)
+ifneq ($(EXTERNAL_KERNEL),YES)
-##	if ! grep -q "$(shell /usr/bin/gcc --version | grep gcc)" "$(shell cat $(KERNEL_PATH))/.config"; then \
+	if ! grep -q "$(shell RUN /usr/bin/gcc --version | grep gcc)" "$(KERNEL_PATH))/.config"; then \
-##		RUN $(EMERGE) $(USEPKG) --onlydeps --oneshot --noreplace sys-kernel/$(KERNEL_PKG); \
+		RUN $(EMERGE) $(USEPKG) --onlydeps --oneshot --noreplace sys-kernel/$(KERNEL_PKG); \
-##		RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) clean oldconfig modules_prepare; \
+		RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) clean oldconfig $(KERNEL_MODULES_PREPARE); \
-##	fi
+	fi
-##endif
+endif
 	touch $(HARDENED)
 
 $(timesyncd_conf): default/timesyncd.conf
@@ -41,9 +42,10 @@ $(tmux_conf): default/tmux.conf
 	cp $< $@
 
 $(PAM_SSH_AGENT_AUTH):
-	sed -i 's/# %wheel ALL=(ALL) ALL/%wheel ALL=(ALL) ALL/' $(CHROOT)/etc/sudoers
+	sed -i 's/# %wheel ALL=(ALL:ALL) NOPASSWD: ALL/%wheel ALL=(ALL:ALL) NOPASSWD: ALL/' $(CHROOT)/etc/sudoers
 	sed -i 's#^auth.*$$#auth sufficient pam_ssh_agent_auth.so file=~/.ssh/authorized_keys\nauth required pam_env.so readenv=1 user_readenv=0\nsession required pam_env.so readenv=1 user_readenv=0\nauth    substack                system-auth#' $(CHROOT)/etc/pam.d/sudo
 	sed -i 's#^auth.*$$#auth sufficient pam_ssh_agent_auth.so file=~/.ssh/authorized_keys\nauth required pam_env.so readenv=1 user_readenv=0\nsession required pam_env.so readenv=1 user_readenv=0\nauth    substack                system-auth#' $(CHROOT)/etc/pam.d/sudo-i
+	mkdir -p $(CHROOT)/etc/sudoers.d
 	echo "Defaults   env_keep += SSH_AUTH_SOCK" > $(CHROOT)/etc/sudoers.d/ssh_auth_sock
 	touch $(PAM_SSH_AGENT_AUTH)
 

appliances/default/default/01firstboot.start

@@ -123,7 +123,7 @@ if [ ! -L "/dev/disk/by-label/$LABEL" ]; then
     exit 1
 fi
 if ! grep -Fq "LABEL=$LABEL" /etc/fstab; then
-    echo "LABEL=$LABEL  /$LABEL  ext4  noatime  0 1" >> /etc/fstab
+    echo -e "LABEL=$LABEL\t/$LABEL\t\text4\tnoatime\t0 1" >> /etc/fstab
 fi
 mount -a
 if ! mount | grep /$LABEL > /dev/null; then
@@ -131,6 +131,21 @@ if ! mount | grep /$LABEL > /dev/null; then
     exit 1
 fi
 
+########################
+# Swap partition
+########################
+if [ -L "/dev/disk/by-label/SWAP" ]; then
+    echo 'Swap partition found, mounting...'
+    swapoff -a
+    grep -Fv swap /etc/fstab > /etc/fstab.tmp && mv /etc/fstab.tmp /etc/fstab
+    echo -e "LABEL=SWAP\tnone\t\tswap\tsw\t0 0" >> /etc/fstab
+    swapon -a
+    rm -f /.swap
+fi
+
+echo 'Mount data partition...'
+
+
 ########################
 # homedirs, users
 ########################

appliances/default/default/bashrc.firstboot

@@ -1,32 +1,38 @@
 if [ -e /01firstboot ]; then
     echo
     echo "##########################################################################"
-    echo "The new or updated appliance still needs to be configured."
+    echo "The new or updated appliance still needs to be initialized."
-    echo "You will be prompted to enter your password for the required root rights."
+    echo "If errors occur, their cause must be fixed. Afterwards the initialization"
-    echo "If errors occur, their cause must be fixed. Afterwards the configuration"
     echo "can be restarted by running \"sudo /usr/local/bin/01firstboot.start\"."
     echo "##########################################################################"
     echo
+    read -n 1 -s -r -p "Press any key to start initialization..."
+    echo
     sudo /usr/local/bin/01firstboot.start
     echo
     echo "##########################################################################"
-    echo "After successful configuration, the appliance should be restarted."
+    echo "After successful initialization, the appliance should be restarted."
+    echo
+    read -n 1 -s -r -p "Press any key to continue..."
     echo
 fi
 
 if [ -e /02firstboot ] && [ ! -e /01firstboot ]; then
     echo
     echo "##########################################################################"
-    echo "After the basic configuration, the special services of the appliance still"
+    echo "After basic initialization, the special services of the appliance still"
-    echo "have to be configured. You may be asked to enter the password for the"
+    echo "have to be configured. Any errors that may occur must be corrected, after"
-    echo "required root rights. Any errors that may occur must be corrected, after"
     echo "which the configuration can be restarted by executing"
     echo "\"sudo /usr/local/bin/02firstboot.start\"."
     echo "##########################################################################"
     echo
+    read -n 1 -s -r -p "Press any key to start configuration..."
+    echo
     sudo /usr/local/bin/02firstboot.start
     echo
     echo "##########################################################################"
     echo "After successful configuration, the appliance should be restarted."
     echo
+    read -n 1 -s -r -p "Press any key to continue..."
+    echo
 fi

appliances/default/make.conf

@@ -5,6 +5,6 @@ MAKEOPTS="-j5"
 ACCEPT_LICENSE="*"
 
 # Python-Version festpinnen (s. auch package.mask)
-USE_PYTHON="3.9"
+USE_PYTHON="3.12"
-PYTHON_TARGETS="python3_9"
+PYTHON_TARGETS="python3_12"
-PYTHON_SINGLE_TARGET="python3_9"
+PYTHON_SINGLE_TARGET="python3_12"

appliances/default/package.mask

@@ -1,2 +1,2 @@
-# Python: nur Version 3.9 verwenden (s. auch make.conf: PYTHON-Variable)
+# Python: nur Version 3.12 verwenden (s. auch make.conf: PYTHON-Variable)
->=dev-lang/python-3.10
+>=dev-lang/python-3.13

appliances/default/world

@@ -1,5 +1,6 @@
 app-admin/logrotate
 app-admin/sudo
+app-arch/p7zip
 app-emulation/open-vm-tools
 app-emulation/qemu-guest-agent
 app-misc/mc

configs/fstab.in

@@ -1,3 +0,0 @@
-# <fs>			<mountpoint>	<type>		<opts>		<dump/pass>
-/dev/DRIVE1		/		ext4		noatime		1 1
-/.swap		none		swap		sw          		0 0

scripts/RUN

@@ -26,16 +26,23 @@ def main():
         ##kernelbind.append(f"--bind={kernelpath}:/usr/src/{os.path.basename(os.path.dirname(kernelpath))}")
         kernelbind.append(f"--bind={kernelpath}:/usr/src/linux")
 
+    loopbind = []
+    if os.path.exists('partitions'):
+        with open('partitions') as f:
+            loopdevice = f.readline().strip('\n')
+            loopbind.append(f"--bind={loopdevice}")
+            loopbind.append(f"--bind={loopdevice}p1")
+
     command = [
         "systemd-nspawn",
         "--quiet",
         f"--directory={environ['CHROOT']}",
         f"--machine={environ['container']}",
-        f"--capability=CAP_NET_ADMIN",
+        f"--capability=CAP_NET_ADMIN,CAP_SYS_RAWIO",
         f"--bind={environ['PORTAGE_DIR']}:/var/db/repos/gentoo",
         f"--bind={environ['VA_PKGDIR']}:/var/cache/binpkgs",
         f"--bind={environ['DISTDIR']}:/var/cache/distfiles",
-    ] + repobind + kernelbind + sys.argv[1:]
+    ] + repobind + kernelbind + loopbind + sys.argv[1:]
 
     if os.environ.get("VA_ARCH") == "linux32":
         command = ["linux32"] + command

scripts/build-kernel

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """Build the latest kernel but only if need be"""
 import os
 import shutil

scripts/fetch-stage3

@@ -6,14 +6,15 @@ import urllib.request
 import os
 import re
 import sys
+import gnupg
 from collections import namedtuple
 
 MIRROR = 'http://gentoo.osuosl.org/'
-Stage3 = namedtuple('Stage3', 'url cpu specialty minimal nomultilib systemd size')
+Stage3 = namedtuple('Stage3', 'url cpu specialty minimal nomultilib systemd mergedusr size')
 MINIMAL = re.compile(r'[\+-]minimal-')
 NOMULTILIB = re.compile(r'[\+-]nomultilib-')
 SYSTEMD = re.compile(r'[\+-]systemd-')
-
+MERGEDUSR = re.compile(r'[\+-]mergedusr-')
 
 def parse_args():
     parser = argparse.ArgumentParser(description=__doc__)
@@ -22,6 +23,7 @@ def parse_args():
     parser.add_argument('--no-multilib', action='store_true', default=False)
     parser.add_argument('--systemd', action='store_true', default=False)
     parser.add_argument('--minimal', action='store_true', default=False)
+    parser.add_argument('--mergedusr', action='store_true', default=False)
     parser.add_argument('--outfile', type=str, default=None)
     parser.add_argument('arch', type=str)
 
@@ -32,9 +34,11 @@ def get_manifest(arch, mirror=MIRROR):
     manifest = []
     url = mirror + '/releases/{0}/autobuilds/latest-stage3.txt'.format(arch)
     base_url = mirror + '/releases/{0}/autobuilds/'.format(arch)
+    gpg = gnupg.GPG(gnupghome="/home/user/.gnupg")
 
     with urllib.request.urlopen(url) as fp:
-        text = fp.read()
+        text = gpg.verify_file(fp, extra_args=['-o', '-']).data
+        ##text = fp.read()
         text = text.decode('ascii')
 
     for line in text.split('\n'):
@@ -57,10 +61,13 @@ def get_manifest(arch, mirror=MIRROR):
         nomultilib = bool(NOMULTILIB.search(rest))
         systemd = 'systemd-' in rest
         systemd = bool(SYSTEMD.search(rest))
+        mergedusr = 'mergedusr-' in rest
+        mergedusr = bool(MERGEDUSR.search(rest))
         minimal = bool(MINIMAL.search(rest))
 
         stage3 = Stage3(
             cpu=cpu,
+            mergedusr=mergedusr,
             minimal=minimal,
             nomultilib=nomultilib,
             systemd=systemd,
@@ -93,11 +100,14 @@ def main():
     stage3 = None
 
     for item in manifest:
+        if not args.cpu and item.cpu != args.arch:
+            continue
         if args.cpu and item.cpu != args.cpu:
             continue
 
         if not all([
             args.minimal == item.minimal,
+            args.mergedusr == item.mergedusr,
             args.no_multilib == item.nomultilib,
             args.systemd == item.systemd,
             args.specialty == item.specialty,

scripts/print

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys