Building a XEN dom0 Linux kernel

The domU instructions below have also turned into a scripts:

Introduction

You need at least Linux v4.2 for the XEN dom0/U code to be available. Slackware 14.2 has v4.4 so that is fine – you could just recompile the sources provided from the K set. But let us get things staight with the latest versions from

Preparing

Grab the latest stable and verify it

version=5.0.9

cd /usr/src/
wget https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-$version.tar.xz
wget https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-$version.tar.sign
unxz linux-$version.tar.xz
gpg2 --recv-keys 0x6092693E --keyserver wwwkeys.pgp.net
gpg2 --verify linux-$version.tar.sign
tar xf linux-$version.tar

#wget https://sourceforge.net/projects/reiser4/files/reiser4-for-linux-4.x/reiser4-for-4.20.1.patch.gz
wget https://sourceforge.net/projects/reiser4/files/reiser4-for-linux-5.x/reiser4-for-5.0.0.patch.gz

or xen testing,

#git clone git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git
#cd tip/
#git reset --hard
#git checkout -b xen-next origin/linux-next
#git pull

differenciate the code bases and apply some patches,

#rm -rf dom0/
#rm -rf domU/
mv linux-$version/ dom0/ && cd dom0/
mv linux-$version/ domU/ && cd domU/
#zcat ../reiser4-for-4.20.1.patch.gz | patch -p1
zcat ../reiser4-for-5.0.0.patch.gz | patch -p1

start clean,

ls -alhF .config*
rm -f .config*
make -j16 distclean
#make -j16 mrproper

and proceed,

mv -i Makefile Makefile.dist
sed 's/^EXTRAVERSION =/EXTRAVERSION = .slackxenreiser4/' Makefile.dist > Makefile
sed 's/^EXTRAVERSION =/EXTRAVERSION = .domureiser4/' Makefile.dist > Makefile

dom0 specific,

wget -O - http://mirror.yandex.ru/slackware/slackware64-current/kernels/huge.s/config > .config
make -j16 olddefconfig

domU specific,

#make help | grep defconfig
#make ARCH=x86_64 tinyconfig
#make ARCH=x86_64 defconfig
make -j16 x86_64_defconfig # same as defconfig

and proceed,

cat >> .config <<-EOF9
CONFIG_BTRFS_FS=y
CONFIG_GFS2_FS=y
CONFIG_JFS_FS=y
CONFIG_REISERFS_FS=y
CONFIG_REISER4_FS=y
CONFIG_XFS_FS=y
CONFIG_BRIDGE=y
CONFIG_VLAN_8021Q=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
CONFIG_NFS_V4_1=y
CONFIG_NFS_V4_2=y
CONFIG_NFS_V4_1_MIGRATION=y
CONFIG_NFSD=y
CONFIG_NFSD_V3=y
CONFIG_NFSD_V4=y
CONFIG_CIFS=y
EOF9
make -j16 olddefconfig
grep REISER4_FS .config

The Docker Situation

cd /usr/src/
#wget https://raw.githubusercontent.com/moby/moby/master/contrib/check-config.sh
#same as wget https://raw.githubusercontent.com/coreos/docker/master/contrib/check-config.sh
wget https://raw.githubusercontent.com/docker/docker/master/contrib/check-config.sh
chmod +x check-config.sh

cd dom0/
cd domU/
../check-config.sh .config | grep missing | sed -r 's/^[[:space:]]*- .*(CONFIG_[^:]+).*: .*missing.*/\1=y/'
vi .config
G

COPY/PASTE into .config

make -j16 olddefconfig && echo DONE
grep VXLAN .config
grep OVERLAY_FS .config

#k8s
cat >> .config <<-EOF9
CONFIG_NETFILTER_ADVANCED=y
CONFIG_IP_NF_TARGET_REDIRECT=y
CONFIG_NETFILTER_XT_MATCH_COMMENT=y
EOF9
make -j16 olddefconfig && echo DONE
grep NETFILTER_ADVANCED .config
grep IP_NF_TARGET_REDIRECT .config
grep NETFILTER_XT_MATCH_COMMENT .config

note. Linux v5 – AUFS_FS is totally gone

XEN

make it generally XEN capable,

make -j16 xenconfig
cat >> .config <<-EOF9
CONFIG_PARAVIRT_SPINLOCKS=y
EOF9
make -j16 olddefconfig && echo DONE
grep CONFIG_XEN_PVH= .config
grep CONFIG_PARAVIRT_SPINLOCKS .config

DOM0 SPECIFIC

cat >> .config <<-EOF9
CONFIG_UFS_FS=y
CONFIG_UFS_FS_WRITE=y
EOF9
make -j16 olddefconfig

grep CEPH .config
grep BLK_DEV_RBD .config
grep INFINIBAND .config
grep INFINIBAND_ISER .config
grep SCSI_LOWLEVEL .config
grep ISCSI_TCP .config
grep BRIDGE_NF_EBTABLES .config
grep BRIDGE_EBT_ .config

TMEM does not seem to work on dom0

xen:tmem: frontswap enabled, RAM provided by Xen Transcendent Memory
xen:tmem: cleancache enabled, RAM provided by Xen Transcendent Memory
xen_selfballoon: Xen selfballooning driver disabled for domain0

DOMU SPECIFIC

domU only & TMEM

grep XEN_DOM0 .config
cat >> .config <<-EOF9
CONFIG_XEN_DOM0=n
CONFIG_XEN_TMEM=y
CONFIG_CLEANCACHE=y
CONFIG_FRONTSWAP=y
CONFIG_SWAP=y
CONFIG_XEN_SELFBALLOONING=y
EOF9
make -j16 olddefconfig

disable as much modules as possible

grep =m$ .config
mv -f .config .config.m
#sed 's/=m$/=n/' .config.m > .config
sed 's/=m$/=y/' .config.m > .config
make -j16 olddefconfig
grep =m$ .config

you will notice those remain

CONFIG_XEN_TMEM=m
CONFIG_PNFS_FLEXFILE_LAYOUT=m

Building & Delivering Modules

time nice make -j16 > ../make.dom0.log && echo DONE
time nice make -j16 > ../make.domU.log && echo DONE

version=`file arch/x86/boot/bzImage | awk '{print $9}'`
echo $version
ls -lhF /lib/modules/
[[ -d /lib/modules/$version/ ]] && rm -rf /lib/modules/$version/ && echo cleaned-up
make -j16 modules_install >/dev/null && echo done
du -sh /lib/modules/$version/ # dom0 207M 210M 209M
du -sh /lib/modules/$version/ # domU 84K / 1.1M
find /lib/modules/$version/
tar czf /data/kernels/lib.modules.$version.tar.gz -C /lib/modules $version/

DOM0 DELIVERY SPECIFIC

depmod -a $version
unset version

ls -lF arch/x86_64/boot/bzImage
ls -lF arch/x86/boot/bzImage # 4.18.20:9747936 4.20:10045920

mv -f /vmlinuz.config /vmlinuz.config.old
mv -f /vmlinuz.map /vmlinuz.map.old
mv -f /vmlinuz /vmlinuz.old
cp -f .config /vmlinuz.config
cp -f System.map /vmlinuz.map
cp arch/x86/boot/bzImage /vmlinuz
ls -lkF /vmlinuz /vmlinuz.old
file /vmlinuz /vmlinuz.old

check that it works,

xl li # no guest
shutdown -r now

and deliver the dom0 kernel onto the farm,

ls -lhF /vmlinuz
ls -lhF /lib/modules/

#cp -f .config /data/kernels/vmlinuz.dom0.config
#cp -f System.map /data/kernels/vmlinuz.dom0.map
#ls -lkF /data/kernels/vmlinuz
#file /data/kernels/vmlinuz

version=`uname -r`

ssh slack2 mv -f /vmlinuz /vmlinuz.old
scp /vmlinuz slack2:/
ssh slack2 ls -lhF /lib/modules/
ssh slack2 "tar xzf /data/kernels/lib.modules.$version.tar.gz -C /lib/modules/ && echo done"
ssh slack2 depmod -a $version

DOMU DELIVERY SPECIFIC

ls -lF arch/x86_64/boot/bzImage
ls -lF arch/x86/boot/bzImage # / 11115792

SHOULD NOT BE A SYMLINK OTHERWISE YOU WILL OVER-WRITE THE TARGET,

ls -lhF /data/kernels/vmlinuz
file /data/kernels/vmlinuz

mv -f /data/kernels/vmlinuz.config /data/kernels/vmlinuz.config.old
mv -f /data/kernels/vmlinuz.map /data/kernels/vmlinuz.map.old
mv -f /data/kernels/vmlinuz /data/kernels/vmlinuz.old
cp -f .config /data/kernels/vmlinuz.config
cp -f System.map /data/kernels/vmlinuz.map
cp -f arch/x86/boot/bzImage /data/kernels/vmlinuz

clean-up from build system

echo /lib/modules/$version/
ls -ldF /lib/modules/$version/
rm -rf /lib/modules/$version/

as XEM_TMEM module keeps being a module, we need to deliver it to the guests to use TMEM,

guest=GUEST

cd /data/guests/$guest/
fsck.ext4 $guest.ext4
mkdir lala/
mount -o loop,rw $guest.ext4 lala/

mkdir -p lala/lib/modules/
ls -alkF lala/lib/modules/
rm -rf lala/lib/modules/*
#for ball in `ls -1 /data/kernels/lib.modules.*.tar.gz`; do
#        echo -n $ball...
#        tar xzf $ball -C lala/lib/modules/ && echo done
#done; unset ball
tar xzf /data/kernels/lib.modules.$version.tar.gz -C lala/lib/modules/
ls -alkF lala/lib/modules/

cat lala/etc/modules
echo tmem >> lala/etc/modules

umount lala/
rmdir lala/
xl create $guest -c

and verify that everything works as expected onto a new guest, including TMEM and Docker,

zcat /proc/config.gz | grep TMEM
zcat /proc/config.gz | grep CLEANCACHE
lsmod | grep tmem
watch free -m

#zcat /proc/config.gz | grep OVERLAY
#docker ps -a

Resources

Trash

domU new school,

ls -lhF /lib/modules/
echo $version
rm -rf /lib/modules/$version/
cp -f .config /data/kernels/vmlinuz.$version.config
cp -f System.map /data/kernels/vmlinuz.$version.map
cp -f arch/x86/boot/bzImage /data/kernels/vmlinuz.$version
ln -sf vmlinuz.$version /data/kernels/vmlinuz

dom0 docker to be added,

CONFIG_NF_NAT_IPV4=y
CONFIG_IP_NF_TARGET_MASQUERADE=y
CONFIG_IP_NF_NAT=y
CONFIG_MEMCG_SWAP_ENABLED=y
#CONFIG_CGROUP_HUGETLB=y
CONFIG_IP_NF_TARGET_REDIRECT=y
CONFIG_EXT3_FS_XATTR=y
CONFIG_AUFS_FS=y

domU docker to be added,

CONFIG_CGROUP_DEVICE=y
CONFIG_MEMCG=y
CONFIG_VETH=y
CONFIG_BRIDGE_NETFILTER=y
CONFIG_NETFILTER_XT_MATCH_IPVS=y
CONFIG_USER_NS=y
CONFIG_CGROUP_PIDS=y
CONFIG_MEMCG_SWAP=y
CONFIG_MEMCG_SWAP_ENABLED=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_THROTTLING=y
CONFIG_CFQ_GROUP_IOSCHED=y
CONFIG_CGROUP_PERF=y
#CONFIG_CGROUP_HUGETLB=y
CONFIG_NET_CLS_CGROUP=y
CONFIG_CGROUP_NET_PRIO=y
CONFIG_CFS_BANDWIDTH=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_IP_NF_TARGET_REDIRECT=y
CONFIG_IP_VS=y
CONFIG_IP_VS_NFCT=y
#CONFIG_IP_VS_PROTO_TCP=y
#CONFIG_IP_VS_PROTO_UDP=y
CONFIG_IP_VS_RR=y
CONFIG_VXLAN=y
CONFIG_INET_ESP=y
CONFIG_INET_XFRM_MODE_TRANSPORT=y
CONFIG_IPVLAN=y
CONFIG_MACVLAN=y
CONFIG_DUMMY=y
CONFIG_NF_NAT_TFTP=y
CONFIG_NF_CONNTRACK_TFTP=y
CONFIG_AUFS_FS=y
CONFIG_BTRFS_FS_POSIX_ACL=y
CONFIG_DM_THIN_PROVISIONING=y
CONFIG_OVERLAY_FS=y

e.g. for dom0,

cat >> .config <<-EOF
CONFIG_NF_NAT_IPV4=y
CONFIG_IP_NF_TARGET_MASQUERADE=y
CONFIG_IP_NF_NAT=y
CONFIG_MEMCG_SWAP_ENABLED=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_EXT3_FS_XATTR=y
CONFIG_AUFS_FS=y
EOF

e.g. for domU,

CONFIG_CGROUP_DEVICE=y
CONFIG_MEMCG=y
CONFIG_VETH=y
CONFIG_BRIDGE_NETFILTER=y
CONFIG_NETFILTER_XT_MATCH_IPVS=y
CONFIG_USER_NS=y
CONFIG_CGROUP_PIDS=y
CONFIG_MEMCG_SWAP=y
CONFIG_MEMCG_SWAP_ENABLED=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_THROTTLING=y
CONFIG_CFQ_GROUP_IOSCHED=y
CONFIG_CGROUP_PERF=y
CONFIG_NET_CLS_CGROUP=y
CONFIG_CGROUP_NET_PRIO=y
CONFIG_CFS_BANDWIDTH=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_IP_VS=y
CONFIG_IP_VS_NFCT=y
CONFIG_IP_VS_RR=y
CONFIG_VXLAN=y
CONFIG_INET_ESP=y
CONFIG_INET_XFRM_MODE_TRANSPORT=y
CONFIG_IPVLAN=y
CONFIG_MACVLAN=y
CONFIG_DUMMY=y
CONFIG_NF_NAT_TFTP=y
CONFIG_NF_CONNTRACK_TFTP=y
CONFIG_AUFS_FS=y
CONFIG_BTRFS_FS_POSIX_ACL=y
CONFIG_DM_THIN_PROVISIONING=y

Nethence | Doc | Pub | Lab | Pbraun | SNE Russia | xhtml