From 81e895368eb7dfc295c7954027b5b051aaed583f Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 30 Jan 2018 14:51:27 +0000 Subject: [PATCH 1/4] Drop duplicate mounts of CNI directories. Previously we had: ``` config: binds: - /var/lib/cni/conf:/etc/cni/net.d:rshared,rbind - /var/lib/cni/bin:/opt/cni/bin:rshared,rbind ... runtime: mounts: - type: bind source: /var/lib/cni/bin destination: /opt/cni/bin options: ["rw","bind"] - type: bind source: /var/lib/cni/conf destination: /etc/cni/net.d options: ["rw","bind"] ``` Which is redundant. Drop `runtime.mounts` to leave just the `binds`. Signed-off-by: Ian Campbell --- pkg/kubelet/build.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pkg/kubelet/build.yml b/pkg/kubelet/build.yml index 1cd77b5..0cdadc2 100644 --- a/pkg/kubelet/build.yml +++ b/pkg/kubelet/build.yml @@ -36,12 +36,3 @@ config: - /var/lib/cni/bin - /var/lib/kubelet-plugins - /var/lib/nfs/statd/sm - mounts: - - type: bind - source: /var/lib/cni/bin - destination: /opt/cni/bin - options: ["rw","bind"] - - type: bind - source: /var/lib/cni/conf - destination: /etc/cni/net.d - options: ["rw","bind"] From 48dcfa1cca651b549a4cafb97e8f1f2e101a38b4 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 30 Jan 2018 15:01:02 +0000 Subject: [PATCH 2/4] Add `findutils` to kubelet container. kubelet wants a fully featured `find` now it seems: ``` E0130 14:50:23.475987 654 fsHandler.go:121] failed to collect filesystem stats - rootDiskErr: , rootInodeErr: cmd [find /var/lib/docker/overlay2/2c4a7692faaaa08afc65430415958e14ae64cd419ad4baa8555cddda3c934814/diff - xdev -printf .] failed. stderr: find: unrecognized: -printf BusyBox v1.27.2 (2017-12-12 10:41:50 GMT) multi-call binary. Usage: find [-HL] [PATH]... [OPTIONS] [ACTIONS] Search for files and perform actions on them. ``` Unsure when this started. Signed-off-by: Ian Campbell --- pkg/kubelet/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kubelet/Dockerfile b/pkg/kubelet/Dockerfile index 8ba3f02..0b3ca6b 100644 --- a/pkg/kubelet/Dockerfile +++ b/pkg/kubelet/Dockerfile @@ -86,6 +86,7 @@ RUN apk add --no-cache --initdb -p /out \ curl \ ebtables \ ethtool \ + findutils \ iproute2 \ iptables \ musl \ From b9bdf91bea733a0d8ed8426db9866bac252e5044 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 30 Jan 2018 16:43:06 +0000 Subject: [PATCH 3/4] kubelet: look for metadata in /run/config. In #53 the `linuxkit/metadata` package was updated, which moved the metadata from `/var/config` to `/run/config`, the effects of which are easy to miss. In my case KUBE_MASTER_UNTAINT=y was having no effect and user pods were therefore not getting scheduld. Signed-off-by: Ian Campbell --- pkg/kubelet/kubeadm-init.sh | 4 ++-- pkg/kubelet/kubelet.sh | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/kubelet/kubeadm-init.sh b/pkg/kubelet/kubeadm-init.sh index ede84bc..e822cc5 100755 --- a/pkg/kubelet/kubeadm-init.sh +++ b/pkg/kubelet/kubeadm-init.sh @@ -12,7 +12,7 @@ else fi # sorting by basename relies on the dirnames having the same number of directories -YAML=$(ls -1 /var/config/kube-system.init/*.yaml /etc/kubeadm/kube-system.init/*.yaml 2>/dev/null | sort --field-separator=/ --key=5) +YAML=$(ls -1 /run/config/kube-system.init/*.yaml /etc/kubeadm/kube-system.init/*.yaml 2>/dev/null | sort --field-separator=/ --key=5) for i in ${YAML}; do n=$(basename "$i") if [ -e "$i" ] ; then @@ -29,7 +29,7 @@ for i in ${YAML}; do fi fi done -if [ -f /var/config/kubeadm/untaint-master ] ; then +if [ -f /run/config/kubeadm/untaint-master ] ; then echo "Removing \"node-role.kubernetes.io/master\" taint from all nodes" kubectl taint nodes --all node-role.kubernetes.io/master- fi diff --git a/pkg/kubelet/kubelet.sh b/pkg/kubelet/kubelet.sh index 409e077..14e5593 100755 --- a/pkg/kubelet/kubelet.sh +++ b/pkg/kubelet/kubelet.sh @@ -6,8 +6,8 @@ if [ -e /etc/kubelet.sh.conf ] ; then . /etc/kubelet.sh.conf fi -if [ -f /var/config/kubelet/disabled ] ; then - echo "kubelet.sh: /var/config/kubelet/disabled file is present, exiting" +if [ -f /run/config/kubelet/disabled ] ; then + echo "kubelet.sh: /run/config/kubelet/disabled file is present, exiting" exit 0 fi if [ -n "$KUBELET_DISABLED" ] ; then @@ -21,9 +21,9 @@ if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then touch /var/lib/cni/.opt.defaults-extracted fi -if [ ! -e /var/lib/cni/.cni.conf-extracted ] && [ -d /var/config/cni ] ; then +if [ ! -e /var/lib/cni/.cni.conf-extracted ] && [ -d /run/config/cni ] ; then mkdir -p /var/lib/cni/conf - cp /var/config/cni/* /var/lib/cni/conf/ + cp /run/config/cni/* /var/lib/cni/conf/ touch /var/lib/cni/.cni.configs-extracted fi @@ -31,20 +31,20 @@ await=/etc/kubernetes/kubelet.conf if [ -f "/etc/kubernetes/kubelet.conf" ] ; then echo "kubelet.sh: kubelet already configured" -elif [ -d /var/config/kubeadm ] ; then - if [ -f /var/config/kubeadm/init ] ; then - echo "kubelet.sh: init cluster with metadata \"$(cat /var/config/kubeadm/init)\"" +elif [ -d /run/config/kubeadm ] ; then + if [ -f /run/config/kubeadm/init ] ; then + echo "kubelet.sh: init cluster with metadata \"$(cat /run/config/kubeadm/init)\"" # This needs to be in the background since it waits for kubelet to start. # We skip printing the token so it is not persisted in the log. - kubeadm-init.sh --skip-token-print $(cat /var/config/kubeadm/init) & - elif [ -e /var/config/kubeadm/join ] ; then - echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/kubeadm/join)\"" - kubeadm join --ignore-preflight-errors=all $(cat /var/config/kubeadm/join) + kubeadm-init.sh --skip-token-print $(cat /run/config/kubeadm/init) & + elif [ -e /run/config/kubeadm/join ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /run/config/kubeadm/join)\"" + kubeadm join --ignore-preflight-errors=all $(cat /run/config/kubeadm/join) await=/etc/kubernetes/bootstrap-kubelet.conf fi -elif [ -e /var/config/userdata ] ; then - echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\"" - kubeadm join --ignore-preflight-errors=all $(cat /var/config/userdata) +elif [ -e /run/config/userdata ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /run/config/userdata)\"" + kubeadm join --ignore-preflight-errors=all $(cat /run/config/userdata) await=/etc/kubernetes/bootstrap-kubelet.conf fi From 270086d472d49c6d0d0363e11888212ec219c792 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 30 Jan 2018 16:49:47 +0000 Subject: [PATCH 4/4] Update hashes Signed-off-by: Ian Campbell --- yml/kube.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yml/kube.yml b/yml/kube.yml index 016bc39..55583de 100644 --- a/yml/kube.yml +++ b/yml/kube.yml @@ -40,7 +40,7 @@ services: image: linuxkit/sshd:4f403fe5ae53dc3e45c8f6972dced9dddf900ae6 cgroupsPath: systemreserved/sshd - name: kubelet - image: linuxkit/kubelet:32dd112401be77a3590a50caf0410aa0ce4d21a9 + image: linuxkit/kubelet:0513947feba7c6cdf5d8d815fb776a99011307ff cgroupsPath: podruntime/kubelet files: - path: etc/linuxkit.yml