virtual-appliance/Makefile

491 lines
17 KiB
Makefile

APPLIANCE ?= base
VABUILDER_OUTPUT := $(CURDIR)
CHROOT := $(VABUILDER_OUTPUT)/build/$(APPLIANCE)
VA_PKGDIR = $(VABUILDER_OUTPUT)/packages
DISTDIR = $(CURDIR)/distfiles
REPO_DIR = $(CURDIR)/repos
REPO_NAMES =
KERNEL_DIR = $(CURDIR)/kernel
PORTAGE_DIR = $(REPO_DIR)/gentoo
HOSTNAME = $(APPLIANCE)
IMAGES = $(VABUILDER_OUTPUT)/images
RAW_IMAGE = $(IMAGES)/$(APPLIANCE).img
QCOW_IMAGE = $(IMAGES)/$(APPLIANCE).qcow
VMDK_IMAGE = $(IMAGES)/$(APPLIANCE).vmdk
VMDK_TYPE = IDE
OVA_IMAGE = $(IMAGES)/$(APPLIANCE).ova
OVA_NUM_CPUS = 1
OVA_MEM_SIZE = 1024
XVA_IMAGE = $(IMAGES)/$(APPLIANCE).xva
LST_FILE = $(IMAGES)/$(APPLIANCE)-packages.lst
CHECKSUMS = $(IMAGES)/SHA256SUMS
STAGE3 = $(CHROOT)/var/tmp/stage3
COMPILE_OPTIONS = $(CHROOT)/var/tmp/compile_options
SOFTWARE = $(CHROOT)/var/tmp/software
KERNEL_SRC = $(CHROOT)/var/tmp/kernel_src
KERNEL = $(CHROOT)/var/tmp/kernel
GRUB = $(CHROOT)/var/tmp/grub
PREPROOT = $(CHROOT)/var/tmp/preproot
SYSTOOLS = $(CHROOT)/var/tmp/systools
STAGE4_TARBALL = $(VABUILDER_OUTPUT)/images/$(APPLIANCE).tar.xz
VIRTIO = NO
TIMEZONE = UTC
DISK_SIZE = 6.0G
SWAP_SIZE = 30
SWAP_FILE = $(CHROOT)/.swap
VA_ARCH = amd64
KERNEL_CONFIG = configs/kernel.config.$(VA_ARCH)
KERNEL_PATH = $(CHROOT)/usr/src/kernel-path
MAKEOPTS = -j5 -l5.64
ENABLE_SSHD = NO
CHANGE_PASSWORD = YES
HEADLESS = NO
EXTERNAL_KERNEL = NO
PKGLIST = 0
DASH = NO
LOCALE ?= en_US.utf8
M4 = m4
EMERGE = emerge --jobs=$(shell nproc --all)
M4_DEFS = -D HOSTNAME=$(HOSTNAME)
M4C = $(M4) $(M4_DEFS)
USEPKG = --usepkg --binpkg-respect-use=y
RSYNC_MIRROR = rsync://rsync.gtlib.gatech.edu/gentoo/
KERNEL_PKG = gentoo-sources
WORLD = appliances/$(APPLIANCE)/world
WORLD_DEFAULT = appliances/default/world
EXTRA_WORLD =
# /etc/portage targets
base_etc_portage := $(wildcard configs/portage/*)
etc_portage := $(patsubst configs/portage/%,$(CHROOT)/etc/portage/%,$(base_etc_portage))
default_package_files := $(wildcard appliances/default/package.*)
appliance_package_files := $(wildcard appliances/$(APPLIANCE)/package.*)
portage_default_package_files = $(patsubst appliances/default/package.%,$(CHROOT)/etc/portage/package.%/01default,$(default_package_files))
portage_package_files = $(patsubst appliances/$(APPLIANCE)/package.%,$(CHROOT)/etc/portage/package.%/02$(APPLIANCE),$(appliance_package_files))
default_make_conf = $(wildcard appliances/default/make.conf)
appliance_make_conf = $(wildcard appliances/$(APPLIANCE)/make.conf)
portage_make_conf_local = $(CHROOT)/etc/portage/make.conf.local
appliance_profile := default/linux/amd64/23.0/systemd
# Allow appliance to override variables
-include appliances/default/default.cfg
-include appliances/$(APPLIANCE)/$(APPLIANCE).cfg
# Allow user to override variables
-include $(profile).cfg
ifneq ($(profile),)
container = $(profile)-$(APPLIANCE)-build
else
container = $(APPLIANCE)-build
endif
PATH := $(CURDIR)/scripts:$(PATH)
export PATH CHROOT container PORTAGE_DIR REPO_DIR REPO_NAMES VA_PKGDIR DISTDIR VA_ARCH
inroot := systemd-nspawn --quiet \
--directory=$(CHROOT) \
--machine=$(container) \
--capability=CAP_NET_ADMIN \
--bind=$(PORTAGE_DIR):/var/db/repos/gentoo \
$(foreach repo,$(REPO_NAMES),--bind=$(REPO_DIR)/$(repo):/var/db/repos/$(repo) ) \
--bind=$(VA_PKGDIR):/var/cache/binpkgs \
--bind=$(DISTDIR):/var/cache/distfiles
ifeq ($(VA_ARCH),x86)
inroot := linux32 $(inroot)
endif
stage4-exists := $(wildcard $(STAGE4_TARBALL))
COPY_ARGS = --exclude-from=configs/rsync-excludes
ifeq ($(CHANGE_PASSWORD),YES)
ifdef ROOT_PASSWORD
change_password = RUN usermod --password '$(ROOT_PASSWORD)' root
else
change_password = RUN passwd --delete root
endif
endif
gcc_config = $(inroot) gcc-config 1
ifneq ($(shell grep 'CONFIG_MODULES=y' $(KERNEL_CONFIG)), )
KERNEL_MODULES_PREPARE = modules_prepare
endif
export APPLIANCE ACCEPT_KEYWORDS CHROOT EMERGE HEADLESS M4 M4C inroot gcc_config
export HOSTNAME MAKEOPTS TIMEZONE USEPKG WORLD
export USEPKG RSYNC_MIRROR
export EXTERNAL_KERNEL KERNEL_PKG KERNEL_PATH KERNEL_CONFIG
all: stage4
image: $(RAW_IMAGE)
sync_portage: $(PORTAGE_DIR)
@print Grabbing latest portage
git -C $(PORTAGE_DIR) pull
touch $(PORTAGE_DIR)
$(PORTAGE_DIR):
@print Grabbing the portage tree
git clone --depth=1 https://github.com/gentoo/gentoo.git $(PORTAGE_DIR)
$(CHROOT)/etc/portage/%: configs/portage/%
COPY --recursive $< /etc/portage/
fstab:
@print Create new /etc/fstab
env echo -e "# <fs>\t\t<mountpoint>\t<type>\t<opts>\t<dump/pass>" > $(CHROOT)/etc/fstab
env echo -e "LABEL=ROOT\t/\t\text4\tnoatime\t1 1" >> $(CHROOT)/etc/fstab
env echo -e "/.swap\t\tnone\t\tswap\tsw\t0 0" >> $(CHROOT)/etc/fstab
env echo -e "#LABEL=SWAP\tnone\t\tswap\tsw\t0 0" >> $(CHROOT)/etc/fstab
$(PREPROOT): $(STAGE3) $(PORTAGE_DIR) $(foreach repo,$(REPO_NAMES),$(REPO_DIR)/$(repo)) fstab $(etc_portage) $(portage_default_package_files) $(portage_package_files)
mkdir -p $(VA_PKGDIR) $(DISTDIR)
@print Creating swap file: `basename $(SWAP_FILE)`
dd if=/dev/zero of=$(SWAP_FILE) bs=1M count=$(SWAP_SIZE)
/sbin/mkswap $(SWAP_FILE)
rm -f $(CHROOT)/etc/resolv.conf
COPY -L /etc/resolv.conf /etc/resolv.conf
touch $(PREPROOT)
stage3-$(VA_ARCH).tar.xz:
@print You do not have a stage3 tarball. Consider \"make sync_stage3\"
@exit 1
sync_stage3:
fetch-stage3 --systemd --outfile=stage3-$(VA_ARCH).tar.xz $(VA_ARCH)
$(STAGE3): stage3-$(VA_ARCH).tar.xz configs/stage3-tarball-excludes
mkdir -p $(CHROOT)
ifdef stage4-exists
@print Using stage4 tarball: `basename $(STAGE4_TARBALL)`
tar xpf "$(STAGE4_TARBALL)" -C $(CHROOT)
else
@print Using stage3 tarball
tar xpf stage3-$(VA_ARCH).tar.xz -C $(CHROOT) --exclude-from=configs/stage3-tarball-excludes
endif
rm -f $(CHROOT)/etc/localtime
touch $(STAGE3)
sync_repos: $(foreach repo,$(REPO_NAMES),$(REPO_DIR)/$(repo))
@print Updating the repositories
$(foreach repo,$(REPO_NAMES),git -C $(REPO_DIR)/$(repo) pull;touch $(REPO_DIR)/$(repo);)
$(foreach repo,$(REPO_NAMES),$(REPO_DIR)/$(repo)):
@print Grabbing a repository
git clone --depth=1 $(REPO_URI_$(subst $(REPO_DIR)/,,$@)) $@
$(CHROOT)/etc/portage/repos.conf: $(foreach repo,$(REPO_NAMES),$(REPO_DIR)/$(repo))
$(foreach repo,$(REPO_NAMES),\
echo "[$(repo)]" >>$@;\
echo "location = /var/db/repos/$(repo)" >>$@;\
echo "auto-sync = no" >>$@;\
echo "" >>$@;\
)
touch $@
$(CHROOT)/etc/portage/package.%/01default: appliances/default/package.% $(STAGE3)
mkdir -p `dirname $@`
cp $< $@
$(CHROOT)/etc/portage/package.%/02$(APPLIANCE): appliances/$(APPLIANCE)/package.% $(STAGE3)
mkdir -p `dirname $@`
cp $< $@
$(portage_make_conf_local): $(default_make_conf) $(appliance_make_conf)
if [ -f "$(default_make_conf)" ] ; \
then COPY $(default_make_conf) /etc/portage/make.conf.local; \
fi
if [ -f "$(appliance_make_conf)" ] ; \
then cat $(appliance_make_conf) >> $(portage_make_conf_local); \
fi
touch $@
$(CHROOT)/var/tmp/profile: $(STAGE3)
RUN eselect profile set $(appliance_profile)
touch $@
$(KERNEL_PATH): $(STAGE3) $(KERNEL_CONFIG)
ifneq ($(EXTERNAL_KERNEL),YES)
$(eval kernel_ebuild = $(shell basename `RUN portageq best_visible / $(KERNEL_PKG)`))
$(eval kernel_name = $(shell echo $(kernel_ebuild) | sed -e 's/\(..*\)-sources-\(..*\)/linux-\2-\1/' -e 's/\(..*\)-\(r[0-9]*\)-\(..*\)/\1-\3-\2/'))
$(eval kernel_config_checksum = $(shell sha1sum $(KERNEL_CONFIG) | cut -c -40))
echo -n "$(KERNEL_DIR)/$(kernel_name)/$(kernel_config_checksum)" > $@
else
echo "NONE" > $@
endif
$(KERNEL_SRC): $(KERNEL_PATH)
ifneq ($(EXTERNAL_KERNEL),YES)
if [ ! -f $(shell cat $(KERNEL_PATH))/Makefile ] ; then \
RUN $(EMERGE) --oneshot --noreplace $(USEPKG) sys-kernel/$(KERNEL_PKG); \
cp -a $(CHROOT)/usr/src/linux-*/* $(shell cat $(KERNEL_PATH)); \
RUN $(EMERGE) -C sys-kernel/$(KERNEL_PKG); \
cp $(KERNEL_CONFIG) $(shell cat $(KERNEL_PATH))/.config; \
RUN make -C /usr/src/linux MAKEOPTS=$(MAKEOPTS) oldconfig $(KERNEL_MODULES_PREPARE); \
fi
endif
touch $(KERNEL_SRC)
$(COMPILE_OPTIONS): $(STAGE3) $(PORTAGE_DIR) configs/make.conf.$(VA_ARCH) configs/locale.gen $(portage_default_package_files) $(portage_package_files) $(portage_make_conf_local) $(CHROOT)/var/tmp/profile $(CHROOT)/etc/portage/repos.conf $(KERNEL_SRC)
COPY configs/make.conf.$(VA_ARCH) /etc/portage/make.conf
COPY configs/locale.gen /etc/locale.gen
RUN locale-gen
touch $(COMPILE_OPTIONS)
$(KERNEL): $(COMPILE_OPTIONS) $(KERNEL_CONFIG) scripts/build-kernel
ifneq ($(EXTERNAL_KERNEL),YES)
@print Configuring kernel
COPY $(KERNEL_CONFIG) /root/kernel.config
COPY scripts/build-kernel /root/build-kernel
RUN --setenv=KERNEL=$(KERNEL_PKG) \
--setenv=EMERGE="$(EMERGE)" \
--setenv=USEPKG="$(USEPKG)" \
--setenv=MAKEOPTS="$(MAKEOPTS)" \
/root/build-kernel
rm -f $(CHROOT)/root/build-kernel
endif
touch $(KERNEL)
$(SYSTOOLS): $(PREPROOT) $(COMPILE_OPTIONS)
@print Installing standard system tools
systemd-firstboot \
--root=$(CHROOT) \
--setup-machine-id \
--timezone=$(TIMEZONE) \
--hostname=$(HOSTNAME) \
--root-password=
RUN eselect locale set $(LOCALE)
ifeq ($(DASH),YES)
if ! test -e "$(STAGE4_TARBALL)"; \
then RUN $(EMERGE) --noreplace $(USEPKG) app-shells/dash; \
echo /bin/dash >> $(CHROOT)/etc/shells; \
RUN chsh -s /bin/sh root; \
fi
RUN ln -sf dash /bin/sh
endif
touch $(SYSTOOLS)
$(GRUB): $(PREPROOT) configs/grub.cfg $(KERNEL) scripts/grub-headless.sed
ifneq ($(EXTERNAL_KERNEL),YES)
@print Installing Grub
RUN $(EMERGE) -nN $(USEPKG) sys-boot/grub
mkdir -p $(CHROOT)/boot/grub
COPY configs/grub.cfg /boot/grub/grub.cfg
ifeq ($(VIRTIO),YES)
sed -i 's/sda/vda/' $(CHROOT)/boot/grub/grub.cfg
endif
ifeq ($(HEADLESS),YES)
sed -i -f scripts/grub-headless.sed $(CHROOT)/boot/grub/grub.cfg
endif
endif
ln -nsf /run/systemd/resolve/resolv.conf $(CHROOT)/etc/resolv.conf
touch $(GRUB)
software: $(SOFTWARE)
$(SOFTWARE): $(SYSTOOLS) configs/eth.network configs/issue $(COMPILE_OPTIONS) $(WORLD)
@print Building $(APPLIANCE)-specific software
if test -f appliances/default/Makefile; \
then $(MAKE) -C appliances/default preinstall; \
fi
$(MAKE) -C appliances/$(APPLIANCE) preinstall
if test -f $(WORLD_DEFAULT); \
then cat $(WORLD_DEFAULT) >> $(CHROOT)/var/lib/portage/world; \
fi
cat $(WORLD) >> $(CHROOT)/var/lib/portage/world
RUN $(EMERGE) $(USEPKG) --update --newuse --deep @system
@print Running @preserved-rebuild
RUN $(EMERGE) --usepkg=n @preserved-rebuild
COPY configs/issue /etc/issue
RUN $(EMERGE) $(USEPKG) --update --newuse --deep @world $(grub_package)
RUN $(EMERGE) --depclean --with-bdeps=n
RUN --setenv EDITOR=/usr/bin/nano etc-update
COPY configs/eth.network /etc/systemd/network/eth.network
RUN systemctl enable systemd-networkd.service
RUN systemctl enable systemd-resolved.service
ifeq ($(ENABLE_SSHD),YES)
RUN systemctl enable sshd.service
endif
ifeq ($(DASH),YES)
RUN $(EMERGE) --depclean app-shells/bash
endif
if test -f appliances/default/Makefile; \
then $(MAKE) -C appliances/default postinstall; \
fi
$(MAKE) -C appliances/$(APPLIANCE) postinstall
ifneq ($(PKGLIST),0)
echo \# > $(LST_FILE)
echo \# Gentoo Virtual Appliance \"$(APPLIANCE)\" package list >> $(LST_FILE)
echo \# Generated `date -u` >> $(LST_FILE)
echo \# >> $(LST_FILE)
(cd "$(CHROOT)"/var/db/pkg ; /bin/ls -1d */* | grep -v '^virtual/') >> $(LST_FILE)
endif
touch $(SOFTWARE)
$(RAW_IMAGE): $(STAGE4_TARBALL) scripts/grub.shell scripts/motd.sh
@print Installing files to `basename $(RAW_IMAGE)`
qemu-img create -f raw $(RAW_IMAGE).tmp $(DISK_SIZE)
parted -s $(RAW_IMAGE).tmp mklabel msdos
parted -s $(RAW_IMAGE).tmp mkpart primary 1 $(DISK_SIZE)
parted -s $(RAW_IMAGE).tmp set 1 boot on
sync
losetup --show --find --partscan $(RAW_IMAGE).tmp > partitions
mkfs.ext4 -O sparse_super,^has_journal -L ROOT -m 0 `cat partitions`p1
mkdir $(CHROOT)
mount -o noatime `cat partitions`p1 $(CHROOT)
tar -xf $(STAGE4_TARBALL) --numeric-owner $(COPY_ARGS) -C $(CHROOT)
motd.sh $(EXTERNAL_KERNEL) $(VIRTIO) $(DISK_SIZE) $(SWAP_SIZE) $(DASH) $(VA_ARCH) > $(CHROOT)/etc/motd
ifneq ($(EXTERNAL_KERNEL),YES)
echo '(hd0) ' `cat partitions` > $(CHROOT)/device-map
cp partitions $(CHROOT)/partitions
RUN /usr/sbin/grub-install --no-floppy --grub-mkdevicemap=device-map --directory=/usr/lib/grub/i386-pc --boot-directory=/boot `cat partitions`
rm $(CHROOT)/device-map $(CHROOT)/partitions
endif
umount $(CHROOT)
rmdir $(CHROOT)
sync
losetup --detach `cat partitions`
rm -f partitions device-map
mv $(RAW_IMAGE).tmp $(RAW_IMAGE)
$(QCOW_IMAGE): $(RAW_IMAGE)
@print Creating `basename $(QCOW_IMAGE)`
qemu-img convert -f raw -O qcow2 -c $(RAW_IMAGE) $(QCOW_IMAGE).tmp
mv $(QCOW_IMAGE).tmp $(QCOW_IMAGE)
qcow: $(QCOW_IMAGE)
$(XVA_IMAGE): $(RAW_IMAGE)
@print Creating `basename $(XVA_IMAGE)`
xva.py --disk=$(RAW_IMAGE) --is-hvm --memory=256 --vcpus=1 --name=$(APPLIANCE) \
--filename=$(XVA_IMAGE).tmp
mv $(XVA_IMAGE).tmp $(XVA_IMAGE)
xva: $(XVA_IMAGE)
$(VMDK_IMAGE): $(RAW_IMAGE)
@print Creating `basename $(VMDK_IMAGE)`
ifeq ($(VMDK_TYPE),SCSI)
qemu-img convert -f raw -O vmdk -o adapter_type=lsilogic,subformat=streamOptimized,hwversion=7 $(RAW_IMAGE) $(VMDK_IMAGE).tmp
else
qemu-img convert -f raw -O vmdk $(RAW_IMAGE) $(VMDK_IMAGE).tmp
endif
mv $(VMDK_IMAGE).tmp $(VMDK_IMAGE)
vmdk: $(VMDK_IMAGE)
NUM_CPUS := $(OVA_NUM_CPUS)
MEM_SIZE := $(OVA_MEM_SIZE)
export NUM_CPUS MEM_SIZE
$(OVA_IMAGE): $(VMDK_IMAGE)
@print Creating `basename $(OVA_IMAGE)`
cd $(IMAGES) && mkova.sh $(APPLIANCE) /usr/share/open-vmdk/template-hw7.ovf $(VMDK_IMAGE)
ova: $(OVA_IMAGE)
$(STAGE4_TARBALL): $(PORTAGE_DIR) stage3-$(VA_ARCH).tar.xz appliances/$(APPLIANCE) configs/rsync-excludes
$(MAKE) $(STAGE3)
$(MAKE) $(PREPROOT)
$(MAKE) $(SOFTWARE)
$(MAKE) $(KERNEL)
$(MAKE) $(GRUB)
@print Creating stage4 tarball: `basename $(STAGE4_TARBALL)`
$(change_password)
mkdir -p $(IMAGES)
tar -acf "$(STAGE4_TARBALL).tmp.xz" --numeric-owner $(COPY_ARGS) -C $(CHROOT) --one-file-system .
mv "$(STAGE4_TARBALL).tmp.xz" "$(STAGE4_TARBALL)"
$(MAKE) clean
stage4: $(STAGE4_TARBALL)
eclean: $(COMPILE_OPTIONS)
RUN $(EMERGE) $(USEPKG) --oneshot --noreplace app-portage/gentoolkit
RUN eclean-pkg
RUN eclean-dist
RUN $(EMERGE) --depclean app-portage/gentoolkit
$(MAKE) clean
clean:
rm -f partitions device-map $(IMAGES)/*.tmp
rm -rf --one-file-system -- $(CHROOT)
realclean: clean
${RM} $(RAW_IMAGE) $(QCOW_IMAGE) $(VMDK_IMAGE)
distclean:
rm -f -- *.qcow *.img *.vmdk
rm -f stage3-*.tar.xz
rm -f portage-snapshot.tar.bz2
appliance-list:
@print 'Available appliances:'
@/bin/ls -1 appliances
checksums:
@print Calculating checksums
$(RM) $(CHECKSUMS)
cd $(IMAGES) && sha256sum --binary * > $(CHECKSUMS).tmp
mv $(CHECKSUMS).tmp $(CHECKSUMS)
shell: $(PREPROOT)
@print 'Entering interactive shell for the $(APPLIANCE) build.'
@print 'Type "exit" or "^D" to leave'
@print
@RUN
@rm -f $(CHROOT)/root/.bash_history
help:
@print 'Help targets (this is not a comprehensive list)'
@echo
@echo 'sync_portage - Download the latest portage snapshot'
@echo 'sync_stage3 - Download the latest stage3 tarball'
@echo 'sync_repos - Download/update the repositories (portage overlays)'
@echo 'stage4 - Build a stage4 tarball'
@echo 'clean - Unmount chroot and clean directory'
@echo 'eclean - Clean outdated packages and distfiles'
@echo 'realclean - Clean and remove image files'
@echo 'shell - Enter a shell in the build environment'
@print 'Images'
@echo 'image - Build a raw VM image from stage4'
@echo 'qcow - Build a qcow VM image from a raw image'
@echo 'vmdk - Build a vmdk image from a raw image'
@echo 'ova - Build a ova image from a vmdk image'
@echo 'xva - Build an xva image from a raw image'
@echo 'appliance-list - List built-in appliances'
@echo 'help - Show this help'
@echo 'Variables'
@echo 'APPLIANCE= - The appliance to build'
@echo 'HOSTNAME= - Hostname to give appliance'
@echo 'TIMEZONE= - Timezone to set for the appliance'
@echo 'CHROOT= - The directory to build the chroot'
@echo 'DISK_SIZE= - Size of the disk image'
@echo 'SWAP_SIZE= - Size of the swap file'
@echo 'VA_ARCH= - Architecture to build for (x86 or amd64)'
@echo 'VIRTIO=YES - Configure the stage2/image to use virtio'
@echo 'EXTERNAL_KERNEL=YES - Do not build a kernel in the image'
@echo 'HEADLESS=YES - Build a headless (serial console) image.'
@echo 'ENABLE_SSHD=YES - Enable sshd to start automatically in the image'
@echo 'VMDK_TYPE=IDE - VMDK image type (IDE or SCSI)'
@echo
@print 'Example to build the base appliance'
@echo 'make APPLIANCE=base HEADLESS=YES VIRTIO=YES stage4 qcow clean'
.PHONY: qcow vmdk ova clean realclean distclean stage4 image stage4 help appliance-list eclean sync_portage sync_stage3 checksums fstab