From 0f910b52519675f60a872c1a919757cd30240085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Deckert?= Date: Sat, 22 May 2021 19:53:34 +0200 Subject: [PATCH] builds the kernel separately to avoid having to rebuild it for any appliance --- Makefile | 31 +++- appliances/default/Makefile | 17 ++- appliances/default/default/01firstboot.start | 148 +++++++++++++++++-- appliances/default/default/bashrc.firstboot | 32 ++++ scripts/RUN | 15 +- scripts/build-kernel | 16 +- 6 files changed, 227 insertions(+), 32 deletions(-) create mode 100644 appliances/default/default/bashrc.firstboot diff --git a/Makefile b/Makefile index 95f9597..375a4e8 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ VA_PKGDIR = $(VABUILDER_OUTPUT)/packages DISTDIR = $(CURDIR)/distfiles REPO_DIR = $(CURDIR)/repos REPO_NAMES = +KERNEL_DIR = $(CURDIR)/kernel PORTAGE_DIR = $(REPO_DIR)/gentoo HOSTNAME = $(APPLIANCE) IMAGES = $(VABUILDER_OUTPUT)/images @@ -21,6 +22,7 @@ CHECKSUMS = $(IMAGES)/SHA256SUMS STAGE3 = $(CHROOT)/var/tmp/stage3 COMPILE_OPTIONS = $(CHROOT)/var/tmp/compile_options SOFTWARE = $(CHROOT)/var/tmp/software +KERNEL_SRC = $(CHROOT)/var/tmp/kernel_src KERNEL = $(CHROOT)/var/tmp/kernel GRUB = $(CHROOT)/var/tmp/grub PREPROOT = $(CHROOT)/var/tmp/preproot @@ -33,6 +35,7 @@ SWAP_SIZE = 30 SWAP_FILE = $(CHROOT)/.swap VA_ARCH = amd64 KERNEL_CONFIG = configs/kernel.config.$(VA_ARCH) +KERNEL_PATH = $(CHROOT)/usr/src/kernel-path MAKEOPTS = -j5 -l5.64 ENABLE_SSHD = NO CHANGE_PASSWORD = YES @@ -112,6 +115,7 @@ gcc_config = $(inroot) gcc-config 1 export APPLIANCE ACCEPT_KEYWORDS CHROOT EMERGE HEADLESS M4 M4C inroot export HOSTNAME MAKEOPTS TIMEZONE USEPKG WORLD export USEPKG RSYNC_MIRROR +export EXTERNAL_KERNEL KERNEL_PKG KERNEL_PATH KERNEL_CONFIG all: stage4 @@ -211,7 +215,29 @@ $(CHROOT)/var/tmp/profile: $(STAGE3) $(CHROOT)/etc/locale.gen: configs/locale.gen COPY configs/locale.gen /etc/locale.gen -$(COMPILE_OPTIONS): $(STAGE3) $(PORTAGE_DIR) $(CHROOT)/etc/portage/make.conf configs/locale.gen $(portage_default_package_files) $(portage_package_files) $(portage_make_conf_local) $(CHROOT)/var/tmp/profile $(CHROOT)/etc/locale.gen $(CHROOT)/etc/portage/repos.conf +$(KERNEL_PATH): $(STAGE3) $(KERNEL_CONFIG) +ifneq ($(EXTERNAL_KERNEL),YES) + $(eval kernel_ebuild = $(shell basename `RUN portageq best_visible / $(KERNEL_PKG)`)) + $(eval kernel_name = $(shell echo $(kernel_ebuild) | sed -e 's/\(..*\)-sources-\(..*\)/linux-\2-\1/' -e 's/\(..*\)-\(r[0-9]*\)-\(..*\)/\1-\3-\2/')) + $(eval kernel_config_checksum = $(shell sha1sum $(KERNEL_CONFIG) | cut -c -40)) + echo -n "$(KERNEL_DIR)/$(kernel_name)/$(kernel_config_checksum)" > $@ +else + echo "NONE" > $@ +endif + +$(KERNEL_SRC): $(KERNEL_PATH) +ifneq ($(EXTERNAL_KERNEL),YES) + if [ ! -f $(shell cat $(KERNEL_PATH))/Makefile ] ; then \ + RUN $(EMERGE) --oneshot --noreplace $(USEPKG) sys-kernel/$(KERNEL_PKG); \ + cp -a $(CHROOT)/usr/src/linux-*/* $(shell cat $(KERNEL_PATH)); \ + RUN $(EMERGE) -C sys-kernel/$(KERNEL_PKG); \ + cp $(KERNEL_CONFIG) $(shell cat $(KERNEL_PATH))/.config; \ + RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) oldconfig modules_prepare; \ + fi +endif + touch $(KERNEL_SRC) + +$(COMPILE_OPTIONS): $(STAGE3) $(PORTAGE_DIR) $(CHROOT)/etc/portage/make.conf configs/locale.gen $(portage_default_package_files) $(portage_package_files) $(portage_make_conf_local) $(CHROOT)/var/tmp/profile $(CHROOT)/etc/locale.gen $(CHROOT)/etc/portage/repos.conf $(KERNEL_SRC) RUN locale-gen touch $(COMPILE_OPTIONS) @@ -274,7 +300,7 @@ $(SOFTWARE): $(SYSTOOLS) configs/eth.network configs/issue $(COMPILE_OPTIONS) $( $(MAKE) -C appliances/$(APPLIANCE) preinstall if test -f $(WORLD_DEFAULT); \ - then COPY $(WORLD_DEFAULT) /var/lib/portage/world; \ + then cat $(WORLD_DEFAULT) >> $(CHROOT)/var/lib/portage/world; \ fi cat $(WORLD) >> $(CHROOT)/var/lib/portage/world RUN $(EMERGE) $(USEPKG) --update --newuse --deep @system @@ -429,6 +455,7 @@ help: @echo @echo 'sync_portage - Download the latest portage snapshot' @echo 'sync_stage3 - Download the latest stage3 tarball' + @echo 'sync_repos - Download/update the repositories (portage overlays)' @echo 'stage4 - Build a stage4 tarball' @echo 'clean - Unmount chroot and clean directory' @echo 'eclean - Clean outdated packages and distfiles' diff --git a/appliances/default/Makefile b/appliances/default/Makefile index 154897b..5cccc64 100644 --- a/appliances/default/Makefile +++ b/appliances/default/Makefile @@ -2,7 +2,7 @@ HARDENED = $(CHROOT)/var/tmp/hardened PAM_SSH_AGENT_AUTH = $(CHROOT)/var/tmp/pam_ssh_agent_auth ADMINUSER = $(CHROOT)/var/tmp/adminuser timesyncd_conf = $(CHROOT)/etc/systemd/timesyncd.conf -01firstboot = $(CHROOT)/etc/local.d/01firstboot.start +01firstboot = $(CHROOT)/usr/local/bin/01firstboot.start network_example = $(CHROOT)/00-eth0.network.example tmux_conf = $(CHROOT)//root/.tmux.conf @@ -11,17 +11,26 @@ $(HARDENED): echo "-hardened" >> $(CHROOT)/etc/portage/profile/use.mask RUN $(EMERGE) $(USEPKG) --oneshot gcc RUN $(EMERGE) $(USEPKG) --oneshot binutils virtual/libc + RUN $(EMERGE) --depclean --with-bdeps=n -$(gcc_config) RUN $(EMERGE) $(USEPKG) --emptytree @world + RUN $(EMERGE) --depclean --with-bdeps=n RUN bash -c 'yes YES | etc-update --automode -9' +ifneq ($(EXTERNAL_KERNEL),YES) + if ! grep -q "$(shell /usr/bin/gcc --version | grep gcc)" "$(shell cat $(KERNEL_PATH))/.config"; then \ + RUN $(EMERGE) $(USEPKG) --onlydeps --oneshot --noreplace sys-kernel/$(KERNEL_PKG); \ + RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) clean oldconfig modules_prepare; \ + fi +endif touch $(HARDENED) $(timesyncd_conf): default/timesyncd.conf cp $< $@ -$(01firstboot): default/01firstboot.start - mkdir -p $(CHROOT)/etc/local.d +$(01firstboot): default/01firstboot.start default/bashrc.firstboot + mkdir -p $(CHROOT)/usr/local/bin cp $< $@ + cat default/bashrc.firstboot >> $(CHROOT)/home/admin/.bashrc touch $(CHROOT)/01firstboot $(network_example): default/00-eth0.network @@ -45,4 +54,4 @@ $(ADMINUSER): preinstall: $(HARDENED) -postinstall: $(timesyncd_conf) $(01firstboot) $(network_example) $(tmux_conf) $(PAM_SSH_AGENT_AUTH) $(ADMINUSER) +postinstall: $(PAM_SSH_AGENT_AUTH) $(ADMINUSER) $(timesyncd_conf) $(01firstboot) $(network_example) $(tmux_conf) diff --git a/appliances/default/default/01firstboot.start b/appliances/default/default/01firstboot.start index eaf152e..ebcd67b 100755 --- a/appliances/default/default/01firstboot.start +++ b/appliances/default/default/01firstboot.start @@ -1,9 +1,102 @@ #!/bin/bash - +# +# This script is called the first time the appliance is started +# after the installation or update. +# +######################## # variables +######################## LABEL="DATA" +APPLCTRL="/$LABEL/.APPLIANCE" +LISTCOPY="$APPLCTRL/populate-data/COPY" +LISTLINK="$APPLCTRL/populate-data/LINK" +######################## +# functions +######################## +copy_data () { + if [ -e "/$1.orig" ]; then + echo "skip /$1 (/$1.orig exists)" + return 0 + fi + if [ -e "/$1" ]; then + # destination exists, copy it to *.orig + cp -af "/$1" "/$1.orig" + else + # destination doesn't exists, create empty *.orig + if [ -d "/$LABEL/$1" ]; then + mkdir -p "/$1.orig" + else + mkdir -p $(dirname /$1) + touch "/$1.orig" + fi + fi + echo "copy /$LABEL/$1" + if [ -d "/$LABEL/$1" ] && [ -d "/$1" ]; then + # Source is a directory and the destination directory exists + cp -af "/$LABEL/$1/." "/$1" + else + mkdir -p $(dirname /$1) + cp -af "/$LABEL/$1" "/$1" + fi +} + +link_data () { + if [ -e "/$1.orig" ]; then + echo "skip /$1 (/$1.orig exists)" + return 0 + fi + if [ -e "/$1" ]; then + # destination exists, move it to *.orig + mv "/$1" "/$1.orig" + else + # destination doesn't exists, create empty *.orig + if [ -d "/$LABEL/$1" ]; then + mkdir -p "/$1.orig" + else + mkdir -p $(dirname /$1) + touch "/$1.orig" + fi + fi + echo "link /$LABEL/$1" + ln -nsf "/$LABEL/$1" "/$1" +} + +populate_data () { + # All files and directories listed in LISTCOPY will be copied from + # /DATA to the system (directories recursively). The files and + # directories listed in LISTLINK will be linked into the system. + # + # Copy + if [ -f "$LISTCOPY" ]; then + while IFS="" read -r src || [ -n "$src" ] + do + [[ -z "$src" ]] && continue + [[ "$src" =~ ^#.*$ ]] && continue + src=${src#/} + [[ "$src" =~ ^home/ ]] && continue + [[ ! -e "/$LABEL/$src" ]] && continue + copy_data "$src" + done < "$LISTCOPY" + fi + + # Link + if [ -f "$LISTLINK" ]; then + while IFS="" read -r src || [ -n "$src" ] + do + [[ -z "$src" ]] && continue + [[ "$src" =~ ^#.*$ ]] && continue + src=${src#/} + [[ "$src" =~ ^home/ ]] && continue + [[ ! -e "/$LABEL/$src" ]] && continue + link_data "$src" + done < "$LISTLINK" + fi +} + +######################## # base settings +######################## set -e [ -e /01firstboot ] || exit 0 @@ -14,7 +107,12 @@ localectl --no-convert set-keymap de-latin1-nodeadkeys echo 'Activate NTP service...' timedatectl set-ntp true +echo 'set machine id...' +systemd-machine-id-setup + +######################## # Data partition +######################## echo 'Mount data partition...' mkdir -p /$LABEL if [ ! -L "/dev/disk/by-label/$LABEL" ]; then @@ -33,6 +131,9 @@ if ! mount | grep /$LABEL > /dev/null; then exit 1 fi +######################## +# homedirs, users +######################## if [ -d "/$LABEL/home" ]; then cd "/$LABEL/home" for user in *; do @@ -46,33 +147,48 @@ if [ -d "/$LABEL/home" ]; then done fi -if [ -d "/$LABEL/etc/ssh" ]; then - rm -rf /etc/ssh - ln -nsf "/$LABEL/etc/ssh" /etc/ssh - systemctl restart sshd -else +######################## +# DATA +######################## + +populate_data + +if [ ! -d "/$LABEL/etc/ssh" ]; then + cp -af /etc/ssh /etc/ssh.orig mkdir -p "/$LABEL/etc" mv /etc/ssh "/$LABEL/etc/" ln -nsf "/$LABEL/etc/ssh" /etc/ssh fi -if [ -f "/$LABEL/etc/hosts" ]; then - rm -f /etc/hosts - ln -nsf "/$LABEL/etc/hosts" /etc/hosts -else +if [ ! -f "/$LABEL/etc/hosts" ]; then + cp -af /etc/hosts /etc/hosts.orig mkdir -p "/$LABEL/etc" mv -f /etc/hosts "/$LABEL/etc/hosts" - cp -f "/$LABEL/etc/hosts" "/$LABEL/etc/hosts.orig" ln -nsf "/$LABEL/etc/hosts" /etc/hosts fi -if [ -f "/$LABEL/etc/fstab" ]; then - cp -f "/$LABEL/etc/fstab" /etc/fstab +if [ ! -f "/$LABEL/etc/fstab" ]; then + cp -af /etc/fstab "/$LABEL/etc/fstab" fi -if [ -d "/$LABEL/etc/systemd" ]; then - cp -a "/$LABEL/etc/systemd" /etc/ - systemctl daemon-reload +if [ ! -f "/$LABEL/etc/hostname" ]; then + cp -af /etc/hostname "/$LABEL/etc/hostname" +fi + +if [ ! -f "$LISTCOPY" ]; then + mkdir -p $(dirname $LISTCOPY) + echo "# all files and directories listed here are copied from /DATA to the system," >> $LISTCOPY + echo "# directories recursively (one file / directory per line)" >> $LISTCOPY + echo "/etc/fstab" >> $LISTCOPY + echo "/etc/hostname" >> $LISTCOPY +fi + +if [ ! -f "$LISTLINK" ]; then + mkdir -p $(dirname $LISTLINK) + echo "# all files and directories listed here are linked from /DATA to the system," >> $LISTLINK + echo "# directories recursively (one file / directory per line)" >> $LISTLINK + echo "/etc/hosts" >> $LISTLINK + echo "/etc/ssh" >> $LISTLINK fi rm /01firstboot diff --git a/appliances/default/default/bashrc.firstboot b/appliances/default/default/bashrc.firstboot new file mode 100644 index 0000000..c5600c3 --- /dev/null +++ b/appliances/default/default/bashrc.firstboot @@ -0,0 +1,32 @@ +if [ -e /01firstboot ]; then + echo + echo "##########################################################################" + echo "The new or updated appliance still needs to be configured." + echo "You will be prompted to enter your password for the required root rights." + echo "If errors occur, their cause must be fixed. Afterwards the configuration" + echo "can be restarted by running \"sudo /usr/local/bin/01firstboot.start\"." + echo "##########################################################################" + echo + sudo /usr/local/bin/01firstboot.start + echo + echo "##########################################################################" + echo "After successful configuration, the appliance should be restarted." + echo +fi + +if [ -e /02firstboot ] && [ ! -e /01firstboot ]; then + echo + echo "##########################################################################" + echo "After the basic configuration, the special services of the appliance still" + echo "have to be configured. You may be asked to enter the password for the" + echo "required root rights. Any errors that may occur must be corrected, after" + echo "which the configuration can be restarted by executing" + echo "\"sudo /usr/local/bin/02firstboot.start\"." + echo "##########################################################################" + echo + sudo /usr/local/bin/02firstboot.start + echo + echo "##########################################################################" + echo "After successful configuration, the appliance should be restarted." + echo +fi diff --git a/scripts/RUN b/scripts/RUN index 94ce7c7..b44593e 100755 --- a/scripts/RUN +++ b/scripts/RUN @@ -13,6 +13,19 @@ def main(): for repo in repos: repobind.append(f"--bind={repodir}/{repo}:/var/db/repos/{repo}") + kernelbind = [] + kernelpathfile=environ['KERNEL_PATH'] + try: + with open(kernelpathfile, 'rb') as f: + kernelpath = f.read().decode("utf-8") + except OSError: + kernelpath = "NONE" + if kernelpath != "NONE": + if not os.path.exists(kernelpath): + os.makedirs(kernelpath) + ##kernelbind.append(f"--bind={kernelpath}:/usr/src/{os.path.basename(os.path.dirname(kernelpath))}") + kernelbind.append(f"--bind={kernelpath}:/usr/src/linux") + command = [ "systemd-nspawn", "--quiet", @@ -22,7 +35,7 @@ def main(): f"--bind={environ['PORTAGE_DIR']}:/var/db/repos/gentoo", f"--bind={environ['VA_PKGDIR']}:/var/cache/binpkgs", f"--bind={environ['DISTDIR']}:/var/cache/distfiles", - ] + repobind + sys.argv[1:] + ] + repobind + kernelbind + sys.argv[1:] if os.environ.get("VA_ARCH") == "linux32": command = ["linux32"] + command diff --git a/scripts/build-kernel b/scripts/build-kernel index 0399852..768d474 100755 --- a/scripts/build-kernel +++ b/scripts/build-kernel @@ -36,10 +36,10 @@ def get_current_kernel(): return version -def install_kernel_package(): +def install_kernel_package_deps(): kernel = os.environ["KERNEL"] package_name = "sys-kernel/{}".format(kernel) - cmd = EMERGE + USEPKG + ["--oneshot", "--noreplace", package_name] + cmd = EMERGE + USEPKG + ["--onlydeps", "--oneshot", "--noreplace", package_name] subprocess.check_call(cmd) @@ -71,7 +71,7 @@ def remove_old_kernels(): if os.path.exists("/boot/vmlinuz"): os.unlink("/boot/vmlinuz") - shutil.rmtree("/lib/modules", ignore_errors=True) + ##shutil.rmtree("/lib/modules", ignore_errors=True) def install_kernel(): @@ -104,9 +104,7 @@ def install_kernel(): break -def uninstall_kernel_package(): - subprocess.check_call(["make", "-C", "/usr/src/linux", "distclean"]) - +def uninstall_kernel_package_deps(): subprocess.check_call(EMERGE + USEPKG + ["--depclean", "--with-bdeps=n"]) @@ -121,14 +119,14 @@ def main(): if current_kernel == latest_kernel: return - install_kernel_package() - copy_kernel_config() + install_kernel_package_deps() + ##copy_kernel_config() build_kernel() remove_old_kernels() install_kernel() backup_kernel_config() - uninstall_kernel_package() + uninstall_kernel_package_deps() main()