builds the kernel separately to avoid having to rebuild it for any appliance

Makefile

@@ -5,6 +5,7 @@ VA_PKGDIR = $(VABUILDER_OUTPUT)/packages
 DISTDIR = $(CURDIR)/distfiles
 REPO_DIR = $(CURDIR)/repos
 REPO_NAMES = 
+KERNEL_DIR = $(CURDIR)/kernel
 PORTAGE_DIR = $(REPO_DIR)/gentoo
 HOSTNAME = $(APPLIANCE)
 IMAGES = $(VABUILDER_OUTPUT)/images
@@ -21,6 +22,7 @@ CHECKSUMS = $(IMAGES)/SHA256SUMS
 STAGE3 = $(CHROOT)/var/tmp/stage3
 COMPILE_OPTIONS = $(CHROOT)/var/tmp/compile_options
 SOFTWARE = $(CHROOT)/var/tmp/software
+KERNEL_SRC = $(CHROOT)/var/tmp/kernel_src
 KERNEL = $(CHROOT)/var/tmp/kernel
 GRUB = $(CHROOT)/var/tmp/grub
 PREPROOT = $(CHROOT)/var/tmp/preproot
@@ -33,6 +35,7 @@ SWAP_SIZE = 30
 SWAP_FILE = $(CHROOT)/.swap
 VA_ARCH = amd64
 KERNEL_CONFIG = configs/kernel.config.$(VA_ARCH)
+KERNEL_PATH = $(CHROOT)/usr/src/kernel-path
 MAKEOPTS = -j5 -l5.64
 ENABLE_SSHD = NO
 CHANGE_PASSWORD = YES
@@ -112,6 +115,7 @@ gcc_config = $(inroot) gcc-config 1
 export APPLIANCE ACCEPT_KEYWORDS CHROOT EMERGE HEADLESS M4 M4C inroot
 export HOSTNAME MAKEOPTS TIMEZONE USEPKG WORLD 
 export USEPKG RSYNC_MIRROR
+export EXTERNAL_KERNEL KERNEL_PKG KERNEL_PATH KERNEL_CONFIG
 
 all: stage4
 
@@ -211,7 +215,29 @@ $(CHROOT)/var/tmp/profile: $(STAGE3)
 $(CHROOT)/etc/locale.gen: configs/locale.gen
 	COPY configs/locale.gen /etc/locale.gen
 
-$(COMPILE_OPTIONS): $(STAGE3) $(PORTAGE_DIR) $(CHROOT)/etc/portage/make.conf configs/locale.gen $(portage_default_package_files) $(portage_package_files) $(portage_make_conf_local) $(CHROOT)/var/tmp/profile $(CHROOT)/etc/locale.gen $(CHROOT)/etc/portage/repos.conf
+$(KERNEL_PATH): $(STAGE3) $(KERNEL_CONFIG)
+ifneq ($(EXTERNAL_KERNEL),YES)
+	$(eval kernel_ebuild = $(shell basename `RUN portageq best_visible / $(KERNEL_PKG)`))
+	$(eval kernel_name = $(shell echo $(kernel_ebuild) | sed -e 's/\(..*\)-sources-\(..*\)/linux-\2-\1/' -e 's/\(..*\)-\(r[0-9]*\)-\(..*\)/\1-\3-\2/'))
+	$(eval kernel_config_checksum = $(shell sha1sum $(KERNEL_CONFIG) | cut -c -40))
+	echo -n "$(KERNEL_DIR)/$(kernel_name)/$(kernel_config_checksum)" > $@
+else
+	echo "NONE" > $@
+endif
+
+$(KERNEL_SRC): $(KERNEL_PATH)
+ifneq ($(EXTERNAL_KERNEL),YES)
+	if [ ! -f $(shell cat $(KERNEL_PATH))/Makefile ] ; then \
+		RUN $(EMERGE) --oneshot --noreplace $(USEPKG) sys-kernel/$(KERNEL_PKG); \
+		cp -a $(CHROOT)/usr/src/linux-*/* $(shell cat $(KERNEL_PATH)); \
+		RUN $(EMERGE) -C sys-kernel/$(KERNEL_PKG); \
+		cp $(KERNEL_CONFIG) $(shell cat $(KERNEL_PATH))/.config; \
+		RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) oldconfig modules_prepare; \
+	fi
+endif
+	touch $(KERNEL_SRC)
+
+$(COMPILE_OPTIONS): $(STAGE3) $(PORTAGE_DIR) $(CHROOT)/etc/portage/make.conf configs/locale.gen $(portage_default_package_files) $(portage_package_files) $(portage_make_conf_local) $(CHROOT)/var/tmp/profile $(CHROOT)/etc/locale.gen $(CHROOT)/etc/portage/repos.conf $(KERNEL_SRC)
 	RUN locale-gen
 	touch $(COMPILE_OPTIONS)
 
@@ -274,7 +300,7 @@ $(SOFTWARE): $(SYSTOOLS) configs/eth.network configs/issue $(COMPILE_OPTIONS) $(
 	$(MAKE) -C appliances/$(APPLIANCE) preinstall
 	
 	if test -f $(WORLD_DEFAULT);  \
-		then COPY $(WORLD_DEFAULT) /var/lib/portage/world; \
+		then cat $(WORLD_DEFAULT) >> $(CHROOT)/var/lib/portage/world; \
 	fi
 	cat $(WORLD) >> $(CHROOT)/var/lib/portage/world
 	RUN $(EMERGE) $(USEPKG) --update --newuse --deep @system
@@ -429,6 +455,7 @@ help:
 	@echo
 	@echo 'sync_portage             - Download the latest portage snapshot'
 	@echo 'sync_stage3              - Download the latest stage3 tarball'
+	@echo 'sync_repos               - Download/update the repositories (portage overlays)'
 	@echo 'stage4                   - Build a stage4 tarball'
 	@echo 'clean                    - Unmount chroot and clean directory'
 	@echo 'eclean                   - Clean outdated packages and distfiles'

appliances/default/Makefile

@@ -2,7 +2,7 @@ HARDENED = $(CHROOT)/var/tmp/hardened
 PAM_SSH_AGENT_AUTH = $(CHROOT)/var/tmp/pam_ssh_agent_auth
 ADMINUSER = $(CHROOT)/var/tmp/adminuser
 timesyncd_conf = $(CHROOT)/etc/systemd/timesyncd.conf
-01firstboot = $(CHROOT)/etc/local.d/01firstboot.start
+01firstboot = $(CHROOT)/usr/local/bin/01firstboot.start
 network_example = $(CHROOT)/00-eth0.network.example
 tmux_conf = $(CHROOT)//root/.tmux.conf
 
@@ -11,17 +11,26 @@ $(HARDENED):
 	echo "-hardened" >> $(CHROOT)/etc/portage/profile/use.mask
 	RUN $(EMERGE) $(USEPKG) --oneshot gcc
 	RUN $(EMERGE) $(USEPKG) --oneshot binutils virtual/libc
+	RUN $(EMERGE) --depclean --with-bdeps=n
 	-$(gcc_config)
 	RUN $(EMERGE) $(USEPKG) --emptytree @world
+	RUN $(EMERGE) --depclean --with-bdeps=n
 	RUN bash -c 'yes YES | etc-update --automode -9'
+ifneq ($(EXTERNAL_KERNEL),YES)
+	if ! grep -q "$(shell /usr/bin/gcc --version | grep gcc)" "$(shell cat $(KERNEL_PATH))/.config"; then \
+		RUN $(EMERGE) $(USEPKG) --onlydeps --oneshot --noreplace sys-kernel/$(KERNEL_PKG); \
+		RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) clean oldconfig modules_prepare; \
+	fi
+endif
 	touch $(HARDENED)
 
 $(timesyncd_conf): default/timesyncd.conf
 	cp $< $@
 
-$(01firstboot): default/01firstboot.start
+$(01firstboot): default/01firstboot.start default/bashrc.firstboot
-	mkdir -p $(CHROOT)/etc/local.d
+	mkdir -p $(CHROOT)/usr/local/bin
 	cp $< $@
+	cat default/bashrc.firstboot >> $(CHROOT)/home/admin/.bashrc
 	touch $(CHROOT)/01firstboot
 
 $(network_example): default/00-eth0.network
@@ -45,4 +54,4 @@ $(ADMINUSER):
 
 preinstall: $(HARDENED)
 
-postinstall: $(timesyncd_conf) $(01firstboot) $(network_example) $(tmux_conf) $(PAM_SSH_AGENT_AUTH) $(ADMINUSER)
+postinstall: $(PAM_SSH_AGENT_AUTH) $(ADMINUSER) $(timesyncd_conf) $(01firstboot) $(network_example) $(tmux_conf)

appliances/default/default/01firstboot.start

@@ -1,9 +1,102 @@
 #!/bin/bash
-
+#
+# This script is called the first time the appliance is started
+# after the installation or update.
+#
+########################
 # variables
+########################
 LABEL="DATA"
+APPLCTRL="/$LABEL/.APPLIANCE"
+LISTCOPY="$APPLCTRL/populate-data/COPY"
+LISTLINK="$APPLCTRL/populate-data/LINK"
 
+########################
+# functions
+########################
+copy_data () {
+    if [ -e "/$1.orig" ]; then
+        echo "skip /$1 (/$1.orig exists)"
+        return 0
+    fi
+    if [ -e "/$1" ]; then
+        # destination exists, copy it to *.orig
+        cp -af "/$1" "/$1.orig"
+    else
+        # destination doesn't exists, create empty *.orig
+        if [ -d "/$LABEL/$1" ]; then
+            mkdir -p "/$1.orig"
+        else
+            mkdir -p $(dirname /$1)
+            touch "/$1.orig"
+        fi
+    fi
+    echo "copy /$LABEL/$1"
+    if [ -d "/$LABEL/$1" ] && [ -d "/$1" ]; then
+            # Source is a directory and the destination directory exists
+            cp -af "/$LABEL/$1/." "/$1"
+    else
+        mkdir -p $(dirname /$1)
+        cp -af "/$LABEL/$1" "/$1"
+    fi
+}
+
+link_data () {
+    if [ -e "/$1.orig" ]; then
+        echo "skip /$1 (/$1.orig exists)"
+        return 0
+    fi
+    if [ -e "/$1" ]; then
+        # destination exists, move it to *.orig
+        mv "/$1" "/$1.orig"
+    else
+        # destination doesn't exists, create empty *.orig
+        if [ -d "/$LABEL/$1" ]; then
+            mkdir -p "/$1.orig"
+        else
+            mkdir -p $(dirname /$1)
+            touch "/$1.orig"
+        fi
+    fi
+    echo "link /$LABEL/$1"
+    ln -nsf "/$LABEL/$1" "/$1"
+}
+
+populate_data () {
+    # All files and directories listed in LISTCOPY will be copied from
+    # /DATA to the system (directories recursively). The files and
+    # directories listed in LISTLINK will be linked into the system.
+    #
+    # Copy
+    if [ -f "$LISTCOPY" ]; then
+        while IFS="" read -r src || [ -n "$src" ]
+        do
+            [[ -z "$src" ]] && continue
+            [[ "$src" =~ ^#.*$ ]] && continue
+            src=${src#/}
+            [[ "$src" =~ ^home/ ]] && continue
+            [[ ! -e "/$LABEL/$src" ]] && continue
+            copy_data "$src"
+        done < "$LISTCOPY"
+    fi
+
+    # Link
+    if [ -f "$LISTLINK" ]; then
+        while IFS="" read -r src || [ -n "$src" ]
+        do
+            [[ -z "$src" ]] && continue
+            [[ "$src" =~ ^#.*$ ]] && continue
+            src=${src#/}
+            [[ "$src" =~ ^home/ ]] && continue
+            [[ ! -e "/$LABEL/$src" ]] && continue
+            link_data "$src"
+        done < "$LISTLINK"
+    fi
+}
+
+########################
 # base settings
+########################
 set -e
 
 [ -e /01firstboot ] || exit 0
@@ -14,7 +107,12 @@ localectl --no-convert set-keymap de-latin1-nodeadkeys
 echo 'Activate NTP service...'
 timedatectl set-ntp true
 
+echo 'set machine id...'
+systemd-machine-id-setup
+
+########################
 # Data partition
+########################
 echo 'Mount data partition...'
 mkdir -p /$LABEL
 if [ ! -L "/dev/disk/by-label/$LABEL" ]; then
@@ -33,6 +131,9 @@ if ! mount | grep /$LABEL > /dev/null; then
     exit 1
 fi
 
+########################
+# homedirs, users
+########################
 if [ -d "/$LABEL/home" ]; then
     cd "/$LABEL/home"
     for user in *; do
@@ -46,33 +147,48 @@ if [ -d "/$LABEL/home" ]; then
     done
 fi
 
-if [ -d "/$LABEL/etc/ssh" ]; then
+########################
-    rm -rf /etc/ssh
+# DATA
-    ln -nsf "/$LABEL/etc/ssh" /etc/ssh
+########################
-    systemctl restart sshd
+
-else
+populate_data
+
+if [ ! -d "/$LABEL/etc/ssh" ]; then
+    cp -af /etc/ssh /etc/ssh.orig
     mkdir -p "/$LABEL/etc"
     mv /etc/ssh "/$LABEL/etc/"
     ln -nsf "/$LABEL/etc/ssh" /etc/ssh
 fi
 
-if [ -f "/$LABEL/etc/hosts" ]; then
+if [ ! -f "/$LABEL/etc/hosts" ]; then
-    rm -f /etc/hosts
+    cp -af /etc/hosts /etc/hosts.orig
-    ln -nsf "/$LABEL/etc/hosts" /etc/hosts
-else
     mkdir -p "/$LABEL/etc"
     mv -f /etc/hosts "/$LABEL/etc/hosts"
-    cp -f "/$LABEL/etc/hosts" "/$LABEL/etc/hosts.orig"
     ln -nsf "/$LABEL/etc/hosts" /etc/hosts
 fi
 
-if [ -f "/$LABEL/etc/fstab" ]; then
+if [ ! -f "/$LABEL/etc/fstab" ]; then
-    cp -f "/$LABEL/etc/fstab" /etc/fstab
+    cp -af /etc/fstab "/$LABEL/etc/fstab"
 fi
 
-if [ -d "/$LABEL/etc/systemd" ]; then
+if [ ! -f "/$LABEL/etc/hostname" ]; then
-    cp -a "/$LABEL/etc/systemd" /etc/
+    cp -af /etc/hostname "/$LABEL/etc/hostname"
-    systemctl daemon-reload
+fi
+
+if [ ! -f "$LISTCOPY" ]; then
+    mkdir -p $(dirname $LISTCOPY)
+    echo "# all files and directories listed here are copied from /DATA to the system," >> $LISTCOPY
+    echo "# directories recursively (one file / directory per line)" >> $LISTCOPY
+    echo "/etc/fstab" >> $LISTCOPY
+    echo "/etc/hostname" >> $LISTCOPY
+fi
+
+if [ ! -f "$LISTLINK" ]; then
+    mkdir -p $(dirname $LISTLINK)
+    echo "# all files and directories listed here are linked from /DATA to the system," >> $LISTLINK
+    echo "# directories recursively (one file / directory per line)" >> $LISTLINK
+    echo "/etc/hosts" >> $LISTLINK
+    echo "/etc/ssh" >> $LISTLINK
 fi
 
 rm /01firstboot

appliances/default/default/bashrc.firstboot

@@ -0,0 +1,32 @@
+if [ -e /01firstboot ]; then
+    echo
+    echo "##########################################################################"
+    echo "The new or updated appliance still needs to be configured."
+    echo "You will be prompted to enter your password for the required root rights."
+    echo "If errors occur, their cause must be fixed. Afterwards the configuration"
+    echo "can be restarted by running \"sudo /usr/local/bin/01firstboot.start\"."
+    echo "##########################################################################"
+    echo
+    sudo /usr/local/bin/01firstboot.start
+    echo
+    echo "##########################################################################"
+    echo "After successful configuration, the appliance should be restarted."
+    echo
+fi
+
+if [ -e /02firstboot ] && [ ! -e /01firstboot ]; then
+    echo
+    echo "##########################################################################"
+    echo "After the basic configuration, the special services of the appliance still"
+    echo "have to be configured. You may be asked to enter the password for the"
+    echo "required root rights. Any errors that may occur must be corrected, after"
+    echo "which the configuration can be restarted by executing"
+    echo "\"sudo /usr/local/bin/02firstboot.start\"."
+    echo "##########################################################################"
+    echo
+    sudo /usr/local/bin/02firstboot.start
+    echo
+    echo "##########################################################################"
+    echo "After successful configuration, the appliance should be restarted."
+    echo
+fi

scripts/RUN

@@ -13,6 +13,19 @@ def main():
     for repo in repos:
         repobind.append(f"--bind={repodir}/{repo}:/var/db/repos/{repo}")
 
+    kernelbind = []
+    kernelpathfile=environ['KERNEL_PATH']
+    try:
+        with open(kernelpathfile, 'rb') as f:
+            kernelpath = f.read().decode("utf-8")
+    except OSError:
+        kernelpath = "NONE"
+    if kernelpath != "NONE":
+        if not os.path.exists(kernelpath):
+            os.makedirs(kernelpath)
+        ##kernelbind.append(f"--bind={kernelpath}:/usr/src/{os.path.basename(os.path.dirname(kernelpath))}")
+        kernelbind.append(f"--bind={kernelpath}:/usr/src/linux")
+
     command = [
         "systemd-nspawn",
         "--quiet",
@@ -22,7 +35,7 @@ def main():
         f"--bind={environ['PORTAGE_DIR']}:/var/db/repos/gentoo",
         f"--bind={environ['VA_PKGDIR']}:/var/cache/binpkgs",
         f"--bind={environ['DISTDIR']}:/var/cache/distfiles",
-    ] + repobind + sys.argv[1:]
+    ] + repobind + kernelbind + sys.argv[1:]
 
     if os.environ.get("VA_ARCH") == "linux32":
         command = ["linux32"] + command

scripts/build-kernel

@@ -36,10 +36,10 @@ def get_current_kernel():
     return version
 
 
-def install_kernel_package():
+def install_kernel_package_deps():
     kernel = os.environ["KERNEL"]
     package_name = "sys-kernel/{}".format(kernel)
-    cmd = EMERGE + USEPKG + ["--oneshot", "--noreplace", package_name]
+    cmd = EMERGE + USEPKG + ["--onlydeps", "--oneshot", "--noreplace", package_name]
 
     subprocess.check_call(cmd)
 
@@ -71,7 +71,7 @@ def remove_old_kernels():
     if os.path.exists("/boot/vmlinuz"):
         os.unlink("/boot/vmlinuz")
 
-    shutil.rmtree("/lib/modules", ignore_errors=True)
+    ##shutil.rmtree("/lib/modules", ignore_errors=True)
 
 
 def install_kernel():
@@ -104,9 +104,7 @@ def install_kernel():
             break
 
 
-def uninstall_kernel_package():
+def uninstall_kernel_package_deps():
-    subprocess.check_call(["make", "-C", "/usr/src/linux", "distclean"])
-
     subprocess.check_call(EMERGE + USEPKG + ["--depclean", "--with-bdeps=n"])
 
 
@@ -121,14 +119,14 @@ def main():
     if current_kernel == latest_kernel:
         return
 
-    install_kernel_package()
+    install_kernel_package_deps()
-    copy_kernel_config()
+    ##copy_kernel_config()
     build_kernel()
     remove_old_kernels()
     install_kernel()
     backup_kernel_config()
 
-    uninstall_kernel_package()
+    uninstall_kernel_package_deps()
 
 
 main()