builds the kernel separately to avoid having to rebuild it for any appliance

This commit is contained in:
Jörg Deckert 2021-05-22 19:53:34 +02:00
parent ba223cd2b9
commit 0f910b5251
6 changed files with 227 additions and 32 deletions

View File

@ -5,6 +5,7 @@ VA_PKGDIR = $(VABUILDER_OUTPUT)/packages
DISTDIR = $(CURDIR)/distfiles DISTDIR = $(CURDIR)/distfiles
REPO_DIR = $(CURDIR)/repos REPO_DIR = $(CURDIR)/repos
REPO_NAMES = REPO_NAMES =
KERNEL_DIR = $(CURDIR)/kernel
PORTAGE_DIR = $(REPO_DIR)/gentoo PORTAGE_DIR = $(REPO_DIR)/gentoo
HOSTNAME = $(APPLIANCE) HOSTNAME = $(APPLIANCE)
IMAGES = $(VABUILDER_OUTPUT)/images IMAGES = $(VABUILDER_OUTPUT)/images
@ -21,6 +22,7 @@ CHECKSUMS = $(IMAGES)/SHA256SUMS
STAGE3 = $(CHROOT)/var/tmp/stage3 STAGE3 = $(CHROOT)/var/tmp/stage3
COMPILE_OPTIONS = $(CHROOT)/var/tmp/compile_options COMPILE_OPTIONS = $(CHROOT)/var/tmp/compile_options
SOFTWARE = $(CHROOT)/var/tmp/software SOFTWARE = $(CHROOT)/var/tmp/software
KERNEL_SRC = $(CHROOT)/var/tmp/kernel_src
KERNEL = $(CHROOT)/var/tmp/kernel KERNEL = $(CHROOT)/var/tmp/kernel
GRUB = $(CHROOT)/var/tmp/grub GRUB = $(CHROOT)/var/tmp/grub
PREPROOT = $(CHROOT)/var/tmp/preproot PREPROOT = $(CHROOT)/var/tmp/preproot
@ -33,6 +35,7 @@ SWAP_SIZE = 30
SWAP_FILE = $(CHROOT)/.swap SWAP_FILE = $(CHROOT)/.swap
VA_ARCH = amd64 VA_ARCH = amd64
KERNEL_CONFIG = configs/kernel.config.$(VA_ARCH) KERNEL_CONFIG = configs/kernel.config.$(VA_ARCH)
KERNEL_PATH = $(CHROOT)/usr/src/kernel-path
MAKEOPTS = -j5 -l5.64 MAKEOPTS = -j5 -l5.64
ENABLE_SSHD = NO ENABLE_SSHD = NO
CHANGE_PASSWORD = YES CHANGE_PASSWORD = YES
@ -112,6 +115,7 @@ gcc_config = $(inroot) gcc-config 1
export APPLIANCE ACCEPT_KEYWORDS CHROOT EMERGE HEADLESS M4 M4C inroot export APPLIANCE ACCEPT_KEYWORDS CHROOT EMERGE HEADLESS M4 M4C inroot
export HOSTNAME MAKEOPTS TIMEZONE USEPKG WORLD export HOSTNAME MAKEOPTS TIMEZONE USEPKG WORLD
export USEPKG RSYNC_MIRROR export USEPKG RSYNC_MIRROR
export EXTERNAL_KERNEL KERNEL_PKG KERNEL_PATH KERNEL_CONFIG
all: stage4 all: stage4
@ -211,7 +215,29 @@ $(CHROOT)/var/tmp/profile: $(STAGE3)
$(CHROOT)/etc/locale.gen: configs/locale.gen $(CHROOT)/etc/locale.gen: configs/locale.gen
COPY configs/locale.gen /etc/locale.gen COPY configs/locale.gen /etc/locale.gen
$(COMPILE_OPTIONS): $(STAGE3) $(PORTAGE_DIR) $(CHROOT)/etc/portage/make.conf configs/locale.gen $(portage_default_package_files) $(portage_package_files) $(portage_make_conf_local) $(CHROOT)/var/tmp/profile $(CHROOT)/etc/locale.gen $(CHROOT)/etc/portage/repos.conf $(KERNEL_PATH): $(STAGE3) $(KERNEL_CONFIG)
ifneq ($(EXTERNAL_KERNEL),YES)
$(eval kernel_ebuild = $(shell basename `RUN portageq best_visible / $(KERNEL_PKG)`))
$(eval kernel_name = $(shell echo $(kernel_ebuild) | sed -e 's/\(..*\)-sources-\(..*\)/linux-\2-\1/' -e 's/\(..*\)-\(r[0-9]*\)-\(..*\)/\1-\3-\2/'))
$(eval kernel_config_checksum = $(shell sha1sum $(KERNEL_CONFIG) | cut -c -40))
echo -n "$(KERNEL_DIR)/$(kernel_name)/$(kernel_config_checksum)" > $@
else
echo "NONE" > $@
endif
$(KERNEL_SRC): $(KERNEL_PATH)
ifneq ($(EXTERNAL_KERNEL),YES)
if [ ! -f $(shell cat $(KERNEL_PATH))/Makefile ] ; then \
RUN $(EMERGE) --oneshot --noreplace $(USEPKG) sys-kernel/$(KERNEL_PKG); \
cp -a $(CHROOT)/usr/src/linux-*/* $(shell cat $(KERNEL_PATH)); \
RUN $(EMERGE) -C sys-kernel/$(KERNEL_PKG); \
cp $(KERNEL_CONFIG) $(shell cat $(KERNEL_PATH))/.config; \
RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) oldconfig modules_prepare; \
fi
endif
touch $(KERNEL_SRC)
$(COMPILE_OPTIONS): $(STAGE3) $(PORTAGE_DIR) $(CHROOT)/etc/portage/make.conf configs/locale.gen $(portage_default_package_files) $(portage_package_files) $(portage_make_conf_local) $(CHROOT)/var/tmp/profile $(CHROOT)/etc/locale.gen $(CHROOT)/etc/portage/repos.conf $(KERNEL_SRC)
RUN locale-gen RUN locale-gen
touch $(COMPILE_OPTIONS) touch $(COMPILE_OPTIONS)
@ -274,7 +300,7 @@ $(SOFTWARE): $(SYSTOOLS) configs/eth.network configs/issue $(COMPILE_OPTIONS) $(
$(MAKE) -C appliances/$(APPLIANCE) preinstall $(MAKE) -C appliances/$(APPLIANCE) preinstall
if test -f $(WORLD_DEFAULT); \ if test -f $(WORLD_DEFAULT); \
then COPY $(WORLD_DEFAULT) /var/lib/portage/world; \ then cat $(WORLD_DEFAULT) >> $(CHROOT)/var/lib/portage/world; \
fi fi
cat $(WORLD) >> $(CHROOT)/var/lib/portage/world cat $(WORLD) >> $(CHROOT)/var/lib/portage/world
RUN $(EMERGE) $(USEPKG) --update --newuse --deep @system RUN $(EMERGE) $(USEPKG) --update --newuse --deep @system
@ -429,6 +455,7 @@ help:
@echo @echo
@echo 'sync_portage - Download the latest portage snapshot' @echo 'sync_portage - Download the latest portage snapshot'
@echo 'sync_stage3 - Download the latest stage3 tarball' @echo 'sync_stage3 - Download the latest stage3 tarball'
@echo 'sync_repos - Download/update the repositories (portage overlays)'
@echo 'stage4 - Build a stage4 tarball' @echo 'stage4 - Build a stage4 tarball'
@echo 'clean - Unmount chroot and clean directory' @echo 'clean - Unmount chroot and clean directory'
@echo 'eclean - Clean outdated packages and distfiles' @echo 'eclean - Clean outdated packages and distfiles'

View File

@ -2,7 +2,7 @@ HARDENED = $(CHROOT)/var/tmp/hardened
PAM_SSH_AGENT_AUTH = $(CHROOT)/var/tmp/pam_ssh_agent_auth PAM_SSH_AGENT_AUTH = $(CHROOT)/var/tmp/pam_ssh_agent_auth
ADMINUSER = $(CHROOT)/var/tmp/adminuser ADMINUSER = $(CHROOT)/var/tmp/adminuser
timesyncd_conf = $(CHROOT)/etc/systemd/timesyncd.conf timesyncd_conf = $(CHROOT)/etc/systemd/timesyncd.conf
01firstboot = $(CHROOT)/etc/local.d/01firstboot.start 01firstboot = $(CHROOT)/usr/local/bin/01firstboot.start
network_example = $(CHROOT)/00-eth0.network.example network_example = $(CHROOT)/00-eth0.network.example
tmux_conf = $(CHROOT)//root/.tmux.conf tmux_conf = $(CHROOT)//root/.tmux.conf
@ -11,17 +11,26 @@ $(HARDENED):
echo "-hardened" >> $(CHROOT)/etc/portage/profile/use.mask echo "-hardened" >> $(CHROOT)/etc/portage/profile/use.mask
RUN $(EMERGE) $(USEPKG) --oneshot gcc RUN $(EMERGE) $(USEPKG) --oneshot gcc
RUN $(EMERGE) $(USEPKG) --oneshot binutils virtual/libc RUN $(EMERGE) $(USEPKG) --oneshot binutils virtual/libc
RUN $(EMERGE) --depclean --with-bdeps=n
-$(gcc_config) -$(gcc_config)
RUN $(EMERGE) $(USEPKG) --emptytree @world RUN $(EMERGE) $(USEPKG) --emptytree @world
RUN $(EMERGE) --depclean --with-bdeps=n
RUN bash -c 'yes YES | etc-update --automode -9' RUN bash -c 'yes YES | etc-update --automode -9'
ifneq ($(EXTERNAL_KERNEL),YES)
if ! grep -q "$(shell /usr/bin/gcc --version | grep gcc)" "$(shell cat $(KERNEL_PATH))/.config"; then \
RUN $(EMERGE) $(USEPKG) --onlydeps --oneshot --noreplace sys-kernel/$(KERNEL_PKG); \
RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) clean oldconfig modules_prepare; \
fi
endif
touch $(HARDENED) touch $(HARDENED)
$(timesyncd_conf): default/timesyncd.conf $(timesyncd_conf): default/timesyncd.conf
cp $< $@ cp $< $@
$(01firstboot): default/01firstboot.start $(01firstboot): default/01firstboot.start default/bashrc.firstboot
mkdir -p $(CHROOT)/etc/local.d mkdir -p $(CHROOT)/usr/local/bin
cp $< $@ cp $< $@
cat default/bashrc.firstboot >> $(CHROOT)/home/admin/.bashrc
touch $(CHROOT)/01firstboot touch $(CHROOT)/01firstboot
$(network_example): default/00-eth0.network $(network_example): default/00-eth0.network
@ -45,4 +54,4 @@ $(ADMINUSER):
preinstall: $(HARDENED) preinstall: $(HARDENED)
postinstall: $(timesyncd_conf) $(01firstboot) $(network_example) $(tmux_conf) $(PAM_SSH_AGENT_AUTH) $(ADMINUSER) postinstall: $(PAM_SSH_AGENT_AUTH) $(ADMINUSER) $(timesyncd_conf) $(01firstboot) $(network_example) $(tmux_conf)

View File

@ -1,9 +1,102 @@
#!/bin/bash #!/bin/bash
#
# This script is called the first time the appliance is started
# after the installation or update.
#
########################
# variables # variables
########################
LABEL="DATA" LABEL="DATA"
APPLCTRL="/$LABEL/.APPLIANCE"
LISTCOPY="$APPLCTRL/populate-data/COPY"
LISTLINK="$APPLCTRL/populate-data/LINK"
########################
# functions
########################
copy_data () {
if [ -e "/$1.orig" ]; then
echo "skip /$1 (/$1.orig exists)"
return 0
fi
if [ -e "/$1" ]; then
# destination exists, copy it to *.orig
cp -af "/$1" "/$1.orig"
else
# destination doesn't exists, create empty *.orig
if [ -d "/$LABEL/$1" ]; then
mkdir -p "/$1.orig"
else
mkdir -p $(dirname /$1)
touch "/$1.orig"
fi
fi
echo "copy /$LABEL/$1"
if [ -d "/$LABEL/$1" ] && [ -d "/$1" ]; then
# Source is a directory and the destination directory exists
cp -af "/$LABEL/$1/." "/$1"
else
mkdir -p $(dirname /$1)
cp -af "/$LABEL/$1" "/$1"
fi
}
link_data () {
if [ -e "/$1.orig" ]; then
echo "skip /$1 (/$1.orig exists)"
return 0
fi
if [ -e "/$1" ]; then
# destination exists, move it to *.orig
mv "/$1" "/$1.orig"
else
# destination doesn't exists, create empty *.orig
if [ -d "/$LABEL/$1" ]; then
mkdir -p "/$1.orig"
else
mkdir -p $(dirname /$1)
touch "/$1.orig"
fi
fi
echo "link /$LABEL/$1"
ln -nsf "/$LABEL/$1" "/$1"
}
populate_data () {
# All files and directories listed in LISTCOPY will be copied from
# /DATA to the system (directories recursively). The files and
# directories listed in LISTLINK will be linked into the system.
#
# Copy
if [ -f "$LISTCOPY" ]; then
while IFS="" read -r src || [ -n "$src" ]
do
[[ -z "$src" ]] && continue
[[ "$src" =~ ^#.*$ ]] && continue
src=${src#/}
[[ "$src" =~ ^home/ ]] && continue
[[ ! -e "/$LABEL/$src" ]] && continue
copy_data "$src"
done < "$LISTCOPY"
fi
# Link
if [ -f "$LISTLINK" ]; then
while IFS="" read -r src || [ -n "$src" ]
do
[[ -z "$src" ]] && continue
[[ "$src" =~ ^#.*$ ]] && continue
src=${src#/}
[[ "$src" =~ ^home/ ]] && continue
[[ ! -e "/$LABEL/$src" ]] && continue
link_data "$src"
done < "$LISTLINK"
fi
}
########################
# base settings # base settings
########################
set -e set -e
[ -e /01firstboot ] || exit 0 [ -e /01firstboot ] || exit 0
@ -14,7 +107,12 @@ localectl --no-convert set-keymap de-latin1-nodeadkeys
echo 'Activate NTP service...' echo 'Activate NTP service...'
timedatectl set-ntp true timedatectl set-ntp true
echo 'set machine id...'
systemd-machine-id-setup
########################
# Data partition # Data partition
########################
echo 'Mount data partition...' echo 'Mount data partition...'
mkdir -p /$LABEL mkdir -p /$LABEL
if [ ! -L "/dev/disk/by-label/$LABEL" ]; then if [ ! -L "/dev/disk/by-label/$LABEL" ]; then
@ -33,6 +131,9 @@ if ! mount | grep /$LABEL > /dev/null; then
exit 1 exit 1
fi fi
########################
# homedirs, users
########################
if [ -d "/$LABEL/home" ]; then if [ -d "/$LABEL/home" ]; then
cd "/$LABEL/home" cd "/$LABEL/home"
for user in *; do for user in *; do
@ -46,33 +147,48 @@ if [ -d "/$LABEL/home" ]; then
done done
fi fi
if [ -d "/$LABEL/etc/ssh" ]; then ########################
rm -rf /etc/ssh # DATA
ln -nsf "/$LABEL/etc/ssh" /etc/ssh ########################
systemctl restart sshd
else populate_data
if [ ! -d "/$LABEL/etc/ssh" ]; then
cp -af /etc/ssh /etc/ssh.orig
mkdir -p "/$LABEL/etc" mkdir -p "/$LABEL/etc"
mv /etc/ssh "/$LABEL/etc/" mv /etc/ssh "/$LABEL/etc/"
ln -nsf "/$LABEL/etc/ssh" /etc/ssh ln -nsf "/$LABEL/etc/ssh" /etc/ssh
fi fi
if [ -f "/$LABEL/etc/hosts" ]; then if [ ! -f "/$LABEL/etc/hosts" ]; then
rm -f /etc/hosts cp -af /etc/hosts /etc/hosts.orig
ln -nsf "/$LABEL/etc/hosts" /etc/hosts
else
mkdir -p "/$LABEL/etc" mkdir -p "/$LABEL/etc"
mv -f /etc/hosts "/$LABEL/etc/hosts" mv -f /etc/hosts "/$LABEL/etc/hosts"
cp -f "/$LABEL/etc/hosts" "/$LABEL/etc/hosts.orig"
ln -nsf "/$LABEL/etc/hosts" /etc/hosts ln -nsf "/$LABEL/etc/hosts" /etc/hosts
fi fi
if [ -f "/$LABEL/etc/fstab" ]; then if [ ! -f "/$LABEL/etc/fstab" ]; then
cp -f "/$LABEL/etc/fstab" /etc/fstab cp -af /etc/fstab "/$LABEL/etc/fstab"
fi fi
if [ -d "/$LABEL/etc/systemd" ]; then if [ ! -f "/$LABEL/etc/hostname" ]; then
cp -a "/$LABEL/etc/systemd" /etc/ cp -af /etc/hostname "/$LABEL/etc/hostname"
systemctl daemon-reload fi
if [ ! -f "$LISTCOPY" ]; then
mkdir -p $(dirname $LISTCOPY)
echo "# all files and directories listed here are copied from /DATA to the system," >> $LISTCOPY
echo "# directories recursively (one file / directory per line)" >> $LISTCOPY
echo "/etc/fstab" >> $LISTCOPY
echo "/etc/hostname" >> $LISTCOPY
fi
if [ ! -f "$LISTLINK" ]; then
mkdir -p $(dirname $LISTLINK)
echo "# all files and directories listed here are linked from /DATA to the system," >> $LISTLINK
echo "# directories recursively (one file / directory per line)" >> $LISTLINK
echo "/etc/hosts" >> $LISTLINK
echo "/etc/ssh" >> $LISTLINK
fi fi
rm /01firstboot rm /01firstboot

View File

@ -0,0 +1,32 @@
if [ -e /01firstboot ]; then
echo
echo "##########################################################################"
echo "The new or updated appliance still needs to be configured."
echo "You will be prompted to enter your password for the required root rights."
echo "If errors occur, their cause must be fixed. Afterwards the configuration"
echo "can be restarted by running \"sudo /usr/local/bin/01firstboot.start\"."
echo "##########################################################################"
echo
sudo /usr/local/bin/01firstboot.start
echo
echo "##########################################################################"
echo "After successful configuration, the appliance should be restarted."
echo
fi
if [ -e /02firstboot ] && [ ! -e /01firstboot ]; then
echo
echo "##########################################################################"
echo "After the basic configuration, the special services of the appliance still"
echo "have to be configured. You may be asked to enter the password for the"
echo "required root rights. Any errors that may occur must be corrected, after"
echo "which the configuration can be restarted by executing"
echo "\"sudo /usr/local/bin/02firstboot.start\"."
echo "##########################################################################"
echo
sudo /usr/local/bin/02firstboot.start
echo
echo "##########################################################################"
echo "After successful configuration, the appliance should be restarted."
echo
fi

View File

@ -13,6 +13,19 @@ def main():
for repo in repos: for repo in repos:
repobind.append(f"--bind={repodir}/{repo}:/var/db/repos/{repo}") repobind.append(f"--bind={repodir}/{repo}:/var/db/repos/{repo}")
kernelbind = []
kernelpathfile=environ['KERNEL_PATH']
try:
with open(kernelpathfile, 'rb') as f:
kernelpath = f.read().decode("utf-8")
except OSError:
kernelpath = "NONE"
if kernelpath != "NONE":
if not os.path.exists(kernelpath):
os.makedirs(kernelpath)
##kernelbind.append(f"--bind={kernelpath}:/usr/src/{os.path.basename(os.path.dirname(kernelpath))}")
kernelbind.append(f"--bind={kernelpath}:/usr/src/linux")
command = [ command = [
"systemd-nspawn", "systemd-nspawn",
"--quiet", "--quiet",
@ -22,7 +35,7 @@ def main():
f"--bind={environ['PORTAGE_DIR']}:/var/db/repos/gentoo", f"--bind={environ['PORTAGE_DIR']}:/var/db/repos/gentoo",
f"--bind={environ['VA_PKGDIR']}:/var/cache/binpkgs", f"--bind={environ['VA_PKGDIR']}:/var/cache/binpkgs",
f"--bind={environ['DISTDIR']}:/var/cache/distfiles", f"--bind={environ['DISTDIR']}:/var/cache/distfiles",
] + repobind + sys.argv[1:] ] + repobind + kernelbind + sys.argv[1:]
if os.environ.get("VA_ARCH") == "linux32": if os.environ.get("VA_ARCH") == "linux32":
command = ["linux32"] + command command = ["linux32"] + command

View File

@ -36,10 +36,10 @@ def get_current_kernel():
return version return version
def install_kernel_package(): def install_kernel_package_deps():
kernel = os.environ["KERNEL"] kernel = os.environ["KERNEL"]
package_name = "sys-kernel/{}".format(kernel) package_name = "sys-kernel/{}".format(kernel)
cmd = EMERGE + USEPKG + ["--oneshot", "--noreplace", package_name] cmd = EMERGE + USEPKG + ["--onlydeps", "--oneshot", "--noreplace", package_name]
subprocess.check_call(cmd) subprocess.check_call(cmd)
@ -71,7 +71,7 @@ def remove_old_kernels():
if os.path.exists("/boot/vmlinuz"): if os.path.exists("/boot/vmlinuz"):
os.unlink("/boot/vmlinuz") os.unlink("/boot/vmlinuz")
shutil.rmtree("/lib/modules", ignore_errors=True) ##shutil.rmtree("/lib/modules", ignore_errors=True)
def install_kernel(): def install_kernel():
@ -104,9 +104,7 @@ def install_kernel():
break break
def uninstall_kernel_package(): def uninstall_kernel_package_deps():
subprocess.check_call(["make", "-C", "/usr/src/linux", "distclean"])
subprocess.check_call(EMERGE + USEPKG + ["--depclean", "--with-bdeps=n"]) subprocess.check_call(EMERGE + USEPKG + ["--depclean", "--with-bdeps=n"])
@ -121,14 +119,14 @@ def main():
if current_kernel == latest_kernel: if current_kernel == latest_kernel:
return return
install_kernel_package() install_kernel_package_deps()
copy_kernel_config() ##copy_kernel_config()
build_kernel() build_kernel()
remove_old_kernels() remove_old_kernels()
install_kernel() install_kernel()
backup_kernel_config() backup_kernel_config()
uninstall_kernel_package() uninstall_kernel_package_deps()
main() main()