From 8a30148702c30558b9e22dec793ca0a7d5ece6ff Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Tue, 5 Dec 2017 10:12:05 +0000
Subject: [PATCH 01/14] CI: Build .iso (but not -efi.iso) images.

Previously (in #8) building both seemed to timeout or otherwise fall foul of
some sort of infra glitch. Try just building one for now.

This is a first step in trying to actually boot images in CI.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 18 ++++++++++++++----
 Makefile             |  2 ++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index c71dcb6..0b12d21 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -35,6 +35,14 @@ linuxkit_pkg_build: &linuxkit_pkg_build
 image_build: &image_build
   docker:
     - image: debian:stretch
+  # image builds seem to need a bit more grunt (RAM) than usual. Possibly getting OOM killed, which https://github.com/moby/tool/pull/191 might help.
+  # NB: This will become a paid for feature at some point soon (with plenty of warning), so is not a long term solution.
+  # small 		1.0 	2GB
+  # medium (default) 	2.0 	4GB	pass:	fail:5
+  # medium+ 		3.0 	6GB	pass:	fail:2
+  # large 		4.0 	8GB	pass:2	fail:
+  # xlarge 		8.0 	16GB
+  resource_class: large
   steps:
     - run:
         name: Configure $PATH
@@ -42,7 +50,7 @@ image_build: &image_build
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
-        command: apt-get update && apt-get install -y ca-certificates curl git make openssh-client
+        command: apt-get update && apt-get install -y ca-certificates curl git make openssh-client strace
     - attach_workspace:
         at: /workspace
     - checkout
@@ -83,9 +91,11 @@ image_build: &image_build
         name: Build images
         command: |
           mkdir -p /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK
-          # KUBE_FORMATS="iso-efi iso-bios" are much slower (especially for RUNTIME=docker) to build than tar.
-          # So for now just build tar files.
-          make KUBE_FORMATS="tar" master node
+          make LINUXKIT_ARGS="-v" KUBE_FORMATS="iso-bios" master node
+          mv kube-master*.iso kube-node*.iso /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK
+    - persist_to_workspace:
+        root: /workspace
+        paths: images
 
 version: 2
 jobs:
diff --git a/Makefile b/Makefile
index 805a8ee..7c3c60e 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,8 @@ KUBE_FORMAT_ARGS := $(patsubst %,-format %,$(KUBE_FORMATS))
 all: master node
 
 master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml
+	# strace logs received signals
+	#strace -f -e trace=signal
 	linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name kube-master $(KUBE_FORMAT_ARGS) $^
 
 node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml

From 40ce9c439c00e5e75d147c7f69d84d5214637d86 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 8 Dec 2017 16:25:32 +0000
Subject: [PATCH 02/14] CI: Try booting something

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 146 ++++++++++++++++++++++++++++---
 test.exp             | 199 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 333 insertions(+), 12 deletions(-)
 create mode 100755 test.exp

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0b12d21..d39db12 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,3 +1,9 @@
+# XXX NOTE:
+#
+#   The images built here will contain an ephemeral/throwaway ssh key
+#   pair, while not a problem to leak (they are per test only) the
+#   resulting images will not fully support ssh.
+
 linuxkit_pkg_build: &linuxkit_pkg_build
   docker:
     - image: debian:stretch
@@ -37,12 +43,13 @@ image_build: &image_build
     - image: debian:stretch
   # image builds seem to need a bit more grunt (RAM) than usual. Possibly getting OOM killed, which https://github.com/moby/tool/pull/191 might help.
   # NB: This will become a paid for feature at some point soon (with plenty of warning), so is not a long term solution.
-  # small 		1.0 	2GB
-  # medium (default) 	2.0 	4GB	pass:	fail:5
-  # medium+ 		3.0 	6GB	pass:	fail:2
-  # large 		4.0 	8GB	pass:2	fail:
-  # xlarge 		8.0 	16GB
-  resource_class: large
+  # small               1.0     2GB
+  # medium (default)    2.0     4GB     pass:   fail:5
+  # medium+             3.0     6GB     pass:   fail:2
+  # large               4.0     8GB     pass:2  fail:
+  # xlarge              8.0     16GB
+  # XXX with linuxkit from https://github.com/linuxkit/linuxkit/pull/2811 then default (medium) should be ok.
+  #resource_class: large
   steps:
     - run:
         name: Configure $PATH
@@ -50,7 +57,7 @@ image_build: &image_build
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
-        command: apt-get update && apt-get install -y ca-certificates curl git make openssh-client strace
+        command: apt-get update && apt-get install -y ca-certificates curl git make openssh-client
     - attach_workspace:
         at: /workspace
     - checkout
@@ -97,6 +104,69 @@ image_build: &image_build
         root: /workspace
         paths: images
 
+image_boot: &image_boot
+  docker:
+    - image: debian:stretch
+  # We are hitting a 30min timeout in kubeadm init when booting with
+  # 3.5G (in medium, the default). Possibly this is just QEMU without
+  # KVM being too slow and it looked a bit like etcd was getting live
+  # locked (was a _huge_ process in local testing).
+  #
+  # Also fails with 1 VCPU and 3.5G in medium and 4 VCPU and 7.5G in large.
+  #
+  # NB: This will become a paid for feature at some point soon (with plenty of warning), so is not a long term solution.
+  # small               1.0     2GB
+  # medium (default)    2.0     4GB
+  # medium+             3.0     6GB
+  # large               4.0     8GB
+  # xlarge              8.0     16GB
+  resource_class: xlarge
+  steps:
+    - run:
+        name: Configure $PATH
+        command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
+    - run:
+        name: Install packages
+        # ca-certificates are needed for attach_workspace (and git over https)
+        command: apt-get update && apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso
+    - attach_workspace:
+        at: /workspace
+    - checkout
+    - run:
+        name: Test boot
+        command: |
+          # Add ssh keys to the image.
+          # TODO could this be integrated into linuxkit/mkimage-iso-*
+          # such that all those options do not need to be repeated
+          # here? e.g. perhaps this could be made to work:
+          # - Create tar with /root/.ssh/authorized_keys=/workspace/ssh/id_rsa.pub in it
+          # - cat $tar | docker run -v /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso:/input.iso linuxkit/mkimage-iso-bios -add-to /input.iso
+          #
+          # Compared with the options used i nmkimage-iso-bios to
+          # build the original dropped -J -joliet-long since xorriso
+          # is more particular about it than genisoimage producing
+          # lots of:
+          #
+          #   Cannot add /bin/ls to Joliet tree. Symlinks can only be added to a Rock Ridge tree.
+
+          xorriso -indev /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso \
+                  -outdev kube-master.iso \
+                  -as genisoimage \
+                    -l -R \
+                    -c isolinux/boot.cat \
+                    -b isolinux/isolinux.bin \
+                    -no-emul-boot \
+                    -boot-load-size 4 \
+                    -boot-info-table \
+                    -input-charset utf8 \
+                  -- \
+                  -pathspecs on \
+                  -add /root/.ssh/authorized_keys=/workspace/ssh/id_rsa.pub
+
+          #cp /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-{master,node}.iso .
+          #cp /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso .
+          SSHOPTS="-i /workspace/ssh/id_rsa" ./test.exp
+
 version: 2
 jobs:
   dependencies:
@@ -108,7 +178,7 @@ jobs:
           command: mkdir -p /workspace/bin
       - run:
           name: Install packages
-          command: apt-get update && apt-get install -y ca-certificates curl
+          command: apt-get update && apt-get install -y ca-certificates curl openssh-client
       - run:
           name: Fetch binaries
           command: |
@@ -129,16 +199,24 @@ jobs:
             e4ca2ef0015a4be8597d31d9e3e70d88da33924ae72b0999e9f3b79304d4710d  /workspace/bin/manifest-tool
             06cd02c4c2e7a3b1ad9899b03b3d4dde5392d964c675247d32f604a24661f839  /workspace/bin/notary
             EOF
+      - run:
+          name: Generate SSH Keys
+          command: |
+            mkdir -p /workspace/ssh
+            ssh-keygen -f /workspace/ssh/id_rsa -C "ephemeral CI ssh key" -N ''
       - run:
           name: Versions
           command: |
-             chmod +x /workspace/bin/docker # docker version deferred until daemon configured in relevant jobs
-             chmod +x /workspace/bin/linuxkit && /workspace/bin/linuxkit version
-             chmod +x /workspace/bin/manifest-tool && /workspace/bin/manifest-tool --version
-             chmod +x /workspace/bin/notary && /workspace/bin/notary version
+            chmod +x /workspace/bin/docker # docker version deferred until daemon configured in relevant jobs
+            chmod +x /workspace/bin/linuxkit && /workspace/bin/linuxkit version
+            chmod +x /workspace/bin/manifest-tool && /workspace/bin/manifest-tool --version
+            chmod +x /workspace/bin/notary && /workspace/bin/notary version
       - persist_to_workspace:
           root: /workspace
           paths: bin
+      - persist_to_workspace:
+          root: /workspace
+          paths: ssh
 
   lint:
     docker:
@@ -240,6 +318,30 @@ jobs:
       - KUBE_RUNTIME: cri-containerd
       - KUBE_NETWORK: bridge
 
+  boot-docker-weave:
+    <<: *image_boot
+    environment:
+      - KUBE_RUNTIME: docker
+      - KUBE_NETWORK: weave
+
+  #boot-docker-bridge:
+  #  <<: *image_boot
+  #  environment:
+  #    - KUBE_RUNTIME: docker
+  #    - KUBE_NETWORK: bridge
+
+  #boot-cri-containerd-weave:
+  #  <<: *image_boot
+  #  environment:
+  #    - KUBE_RUNTIME: cri-containerd
+  #    - KUBE_NETWORK: weave
+
+  boot-cri-containerd-bridge:
+    <<: *image_boot
+    environment:
+      - KUBE_RUNTIME: cri-containerd
+      - KUBE_NETWORK: bridge
+
   push-pkgs-to-hub:
     docker:
       - image: debian:stretch
@@ -336,6 +438,22 @@ workflows:
             - pkg-kubelet
             - pkg-cri-containerd
 
+      - boot-docker-weave:
+          requires:
+            - image-docker-weave
+
+      #- boot-docker-bridge:
+      #    requires:
+      #      - image-docker-bridge
+
+      #- boot-cri-containerd-weave:
+      #    requires:
+      #      - image-cri-containerd-weave
+
+      - boot-cri-containerd-bridge:
+          requires:
+            - image-cri-containerd-bridge
+
       - push-pkgs-to-hub:
           # We want everything to have passed, which is a bit
           # tedious. Some of these are already covered transitively,
@@ -351,3 +469,7 @@ workflows:
             - image-docker-bridge
             - image-cri-containerd-weave
             - image-cri-containerd-bridge
+            - boot-docker-weave
+            #- boot-docker-bridge
+            #- boot-cri-containerd-weave
+            - boot-cri-containerd-bridge
diff --git a/test.exp b/test.exp
new file mode 100755
index 0000000..32eee0e
--- /dev/null
+++ b/test.exp
@@ -0,0 +1,199 @@
+#!/usr/bin/env expect
+
+set con_prompt "(ns: getty) linuxkit-*:*# "
+set ssh_prompt "linuxkit-*:*# "
+set timeout 120
+
+proc kill args {
+    foreach what $args {
+        global $what
+        if [info exists $what] {
+            upvar #0 $what sid
+            set pid [exp_pid -i $sid]
+            puts "killing $what ($sid) = $pid"
+            exec kill $pid
+            close $sid
+        } else {
+            puts "not killing $what (not started)"
+        }
+    }
+}
+
+proc boot_linuxkit {} {
+    global lk_sid
+    # "medium" level circle CI has 4G of RAM, use 3.5G for VM
+    # small               1.0     2GB
+    # medium (default)    2.0     4GB     (3584M)
+    # medium+             3.0     6GB
+    # large               4.0     8GB     (7680M)
+    # xlarge              8.0     16GB    (15827M)
+    spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=15872 KUBE_VCPUS=8 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y ./boot.sh
+    set lk_sid $spawn_id
+    puts "INFO lk ($lk_sid) is pid [exp_pid -i $lk_sid]"
+}
+
+proc ssh_into_kubelet {} {
+    global ssh_sid
+
+    set sshopts {-p 2222 -o ConnectTimeout=5 -o LogLevel=DEBUG}
+    if [info exists ::env(SSHOPTS)] {
+        set sshopts [concat $::env(SSHOPTS) $sshopts]
+    }
+    spawn env SSHOPTS=$sshopts ./ssh_into_kubelet.sh localhost
+    set ssh_sid $spawn_id
+    puts "INFO ssh ($ssh_sid) is pid [exp_pid -i $ssh_sid]"
+}
+
+proc await_prompt {sidvar promptvar step} {
+    upvar #0 $sidvar sid $promptvar prompt
+    expect -i $sid \
+        $prompt {
+            puts "SUCCESS $step"
+        } timeout {
+            puts "FAIL $step (timeout)"
+            kill ssh_sid lk_sid
+            exit 1
+        } eof {
+            puts "FAIL $step (eof)"
+            kill ssh_sid lk_sid
+            exit 1
+        }
+}
+
+proc send_con {s} {
+    global lk_sid
+    send -i $lk_sid $s
+}
+
+proc await_con_prompt {step} {
+    global lk_sid con_prompt
+    await_prompt lk_sid con_prompt $step
+}
+
+proc send_ssh {s} {
+    global ssh_sid
+    send -i $ssh_sid $s
+}
+
+proc await_ssh_prompt {step} {
+    global ssh_sid ssh_prompt
+    await_prompt ssh_sid ssh_prompt $step
+}
+
+boot_linuxkit
+
+await_con_prompt "boot"
+
+send_con "ifconfig eth0\n"
+await_con_prompt "ifconfig"
+
+send_con "ctr container ls\n"
+await_con_prompt "ctr container ls"
+
+send_con "ctr task ls\n"
+# This was added in a debug attempt, but it turned out sshd was failing to come up (which this didn't help us to discover), so not much use now.
+#set retries 0
+#expect -i lk_sid -timeout 300 \
+#    "sshd * RUNNING" {
+#        puts "SUCCESS sshd running"
+#    } $con_prompt {
+#        set retries [expr $retries + 1]
+#        puts "RETRY:$retries await sshd"
+#        sleep 1
+#        send_con "ctr task ls\n"
+#        exp_continue -continue_timer
+#    } timeout {
+#        puts "FAIL sshd (timeout)"
+#        kill ssh_sid lk_sid
+#    }
+#await_con_prompt "post sshd startup (after $retries attempts)"
+
+set retries 0
+ssh_into_kubelet
+# provide ssh_sid as an indirect, allowing ssh to be respawned, which
+# changes the id, we need this in case ssh cannot immediately connect.
+expect -i ssh_sid \
+    $ssh_prompt {
+        puts "SUCCESS connected to ssh (after $retries attempts)"
+    } "read: Connection reset by peer" {
+        # ssh happened too soon, wait a bit.
+        set retries [expr $retries + 1]
+        puts "RETRY:$retries ssh (conn reset)"
+        wait -i $ssh_sid
+        sleep 1
+        ssh_into_kubelet
+        exp_continue -continue_timer
+    } eof {
+        set retries [expr $retries + 1]
+        puts "RETRY:$retries ssh (eof)"
+        wait -i $ssh_sid
+        sleep 1
+        ssh_into_kubelet
+        exp_continue -continue_timer
+    } timeout {
+        puts "FAIL ssh (timeout)"
+        kill ssh_sid lk_sid
+        exit 1
+    }
+
+puts "RUN kubeadm-init.sh"
+send_ssh "kubeadm-init.sh\n"
+
+# Written as N*5m with logging to avoid "10 mins with no output" from CI
+set retries 0
+set maxretries 10
+expect -i $ssh_sid -timeout 300 \
+    "Your Kubernetes master has initialized successfully!" {
+        puts "SUCCESS cluster initialised!"
+    } $ssh_prompt {
+        puts "FAIL kubeadm-init.sh (returned to prompt)"
+        kill ssh_sid lk_sid
+        exit 1
+    } timeout {
+        set retries [expr $retries + 1]
+        if [expr $retries < $maxretries] {
+            puts "RETRY:$retries kubeadm-init.sh (timeout)"
+            exp_continue
+        }
+        puts "FAIL kubeadm-init.sh (timeout)"
+        kill ssh_sid lk_sid
+        exit 1
+    } eof {
+        puts "FAIL kubeadm-init.sh (eof)"
+        kill ssh_sid lk_sid
+        exit 1
+    }
+await_ssh_prompt "kubeadm-init.sh"
+puts "SUCCESS kubeadm-init complete"
+
+send_ssh "kubectl get nodes\n"
+set retries 0
+expect -i lk_sid -timeout 300 \
+    "linuxkit-* Ready" {
+        puts "SUCCESS node ready"
+    } $ssh_prompt {
+        set retries [expr $retries + 1]
+        puts "RETRY:$retries await node ready"
+        sleep 1
+        send_con "kubectl get nodes\n"
+        exp_continue -continue_timer
+    } timeout {
+        puts "FAIL node ready (timeout)"
+        kill ssh_sid lk_sid
+    }
+await_ssh_prompt "post await node ready (after $retries attempts)"
+
+kill ssh_sid
+
+puts "RUN poweroff -f"
+send_con "poweroff -f\n"
+
+expect -i $lk_sid \
+    "Power down" {
+        puts "SUCCESS poweroff"
+    } eof {
+        puts "SUCCESS poweroff"
+    } timeout {
+        puts "FAILED poweroff (timeout)"
+        exit 1
+    }

From 769fa976b63ce3a36ff0231e1987fa7648578603 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 11:11:48 +0000
Subject: [PATCH 03/14] Drop some commented info

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d39db12..c43de55 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,9 +1,3 @@
-# XXX NOTE:
-#
-#   The images built here will contain an ephemeral/throwaway ssh key
-#   pair, while not a problem to leak (they are per test only) the
-#   resulting images will not fully support ssh.
-
 linuxkit_pkg_build: &linuxkit_pkg_build
   docker:
     - image: debian:stretch
@@ -41,15 +35,6 @@ linuxkit_pkg_build: &linuxkit_pkg_build
 image_build: &image_build
   docker:
     - image: debian:stretch
-  # image builds seem to need a bit more grunt (RAM) than usual. Possibly getting OOM killed, which https://github.com/moby/tool/pull/191 might help.
-  # NB: This will become a paid for feature at some point soon (with plenty of warning), so is not a long term solution.
-  # small               1.0     2GB
-  # medium (default)    2.0     4GB     pass:   fail:5
-  # medium+             3.0     6GB     pass:   fail:2
-  # large               4.0     8GB     pass:2  fail:
-  # xlarge              8.0     16GB
-  # XXX with linuxkit from https://github.com/linuxkit/linuxkit/pull/2811 then default (medium) should be ok.
-  #resource_class: large
   steps:
     - run:
         name: Configure $PATH

From 4eec22b927d7417dad52b34427f235411c3482c0 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 11:13:42 +0000
Subject: [PATCH 04/14] Try using machine job type to boot.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index c43de55..cd97418 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -90,22 +90,9 @@ image_build: &image_build
         paths: images
 
 image_boot: &image_boot
-  docker:
-    - image: debian:stretch
-  # We are hitting a 30min timeout in kubeadm init when booting with
-  # 3.5G (in medium, the default). Possibly this is just QEMU without
-  # KVM being too slow and it looked a bit like etcd was getting live
-  # locked (was a _huge_ process in local testing).
-  #
-  # Also fails with 1 VCPU and 3.5G in medium and 4 VCPU and 7.5G in large.
-  #
-  # NB: This will become a paid for feature at some point soon (with plenty of warning), so is not a long term solution.
-  # small               1.0     2GB
-  # medium (default)    2.0     4GB
-  # medium+             3.0     6GB
-  # large               4.0     8GB
-  # xlarge              8.0     16GB
-  resource_class: xlarge
+  machine:
+    enabled: true
+    image: circleci/classic:latest # default Ubuntu 14.04 image
   steps:
     - run:
         name: Configure $PATH

From e9d816f53f1b1241621ad89feb265d3acc8c2fcb Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 11:34:11 +0000
Subject: [PATCH 05/14] less verbose, reduce noise vs master

---
 .circleci/config.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cd97418..cc6cfc4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -83,7 +83,7 @@ image_build: &image_build
         name: Build images
         command: |
           mkdir -p /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK
-          make LINUXKIT_ARGS="-v" KUBE_FORMATS="iso-bios" master node
+          make KUBE_FORMATS="iso-bios" master node
           mv kube-master*.iso kube-node*.iso /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK
     - persist_to_workspace:
         root: /workspace
@@ -179,10 +179,10 @@ jobs:
       - run:
           name: Versions
           command: |
-            chmod +x /workspace/bin/docker # docker version deferred until daemon configured in relevant jobs
-            chmod +x /workspace/bin/linuxkit && /workspace/bin/linuxkit version
-            chmod +x /workspace/bin/manifest-tool && /workspace/bin/manifest-tool --version
-            chmod +x /workspace/bin/notary && /workspace/bin/notary version
+             chmod +x /workspace/bin/docker # docker version deferred until daemon configured in relevant jobs
+             chmod +x /workspace/bin/linuxkit && /workspace/bin/linuxkit version
+             chmod +x /workspace/bin/manifest-tool && /workspace/bin/manifest-tool --version
+             chmod +x /workspace/bin/notary && /workspace/bin/notary version
       - persist_to_workspace:
           root: /workspace
           paths: bin

From 4975d44d39ab2699ce52048bfbb3b4204c75138a Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 11:49:39 +0000
Subject: [PATCH 06/14] use sudo, machine executor does not run as root

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cc6cfc4..35237ef 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -100,13 +100,14 @@ image_boot: &image_boot
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
-        command: apt-get update && apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso
+        command: sudo apt-get update && sudo apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso
     - attach_workspace:
         at: /workspace
     - checkout
     - run:
         name: Test boot
         command: |
+          ls -lrt /dev/kvm* || true
           # Add ssh keys to the image.
           # TODO could this be integrated into linuxkit/mkimage-iso-*
           # such that all those options do not need to be repeated

From ba058d74b5cf09281319dd35f8751f9b6c043dbe Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 13:07:55 +0000
Subject: [PATCH 07/14] Just test cri-container + bridge while debugging CI

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 96 ++++++++++++++++++++++----------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 35237ef..566b36c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -265,37 +265,37 @@ jobs:
   pkg-kubernetes-docker-image-cache-control-plane:
     <<: *linuxkit_pkg_build
 
-  image-docker-weave:
-    <<: *image_build
-    # Needs to be configured/enabled by CircleCI person
-    #resource_class: large
-    environment:
-      - KUBE_RUNTIME: docker
-      - KUBE_NETWORK: weave
-  image-docker-bridge:
-    <<: *image_build
-    # Needs to be configured/enabled by CircleCI person
-    #resource_class: large
-    environment:
-      - KUBE_RUNTIME: docker
-      - KUBE_NETWORK: bridge
+  #image-docker-weave:
+  #  <<: *image_build
+  #  # Needs to be configured/enabled by CircleCI person
+  #  #resource_class: large
+  #  environment:
+  #    - KUBE_RUNTIME: docker
+  #    - KUBE_NETWORK: weave
+  #image-docker-bridge:
+  #  <<: *image_build
+  #  # Needs to be configured/enabled by CircleCI person
+  #  #resource_class: large
+  #  environment:
+  #    - KUBE_RUNTIME: docker
+  #    - KUBE_NETWORK: bridge
 
-  image-cri-containerd-weave:
-    <<: *image_build
-    environment:
-      - KUBE_RUNTIME: cri-containerd
-      - KUBE_NETWORK: weave
+  #image-cri-containerd-weave:
+  #  <<: *image_build
+  #  environment:
+  #    - KUBE_RUNTIME: cri-containerd
+  #    - KUBE_NETWORK: weave
   image-cri-containerd-bridge:
     <<: *image_build
     environment:
       - KUBE_RUNTIME: cri-containerd
       - KUBE_NETWORK: bridge
 
-  boot-docker-weave:
-    <<: *image_boot
-    environment:
-      - KUBE_RUNTIME: docker
-      - KUBE_NETWORK: weave
+  #boot-docker-weave:
+  #  <<: *image_boot
+  #  environment:
+  #    - KUBE_RUNTIME: docker
+  #    - KUBE_NETWORK: weave
 
   #boot-docker-bridge:
   #  <<: *image_boot
@@ -388,32 +388,32 @@ workflows:
           requires:
             - dependencies
 
-      - image-docker-weave:
-          requires:
-            - dependencies
-            - pkg-kubelet
-            - pkg-kubernetes-docker-image-cache-common
-            - pkg-kubernetes-docker-image-cache-control-plane
-      - image-docker-bridge:
-          requires:
-            - dependencies
-            - pkg-kubelet
-            - pkg-kubernetes-docker-image-cache-common
-            - pkg-kubernetes-docker-image-cache-control-plane
-      - image-cri-containerd-weave:
-          requires:
-            - dependencies
-            - pkg-kubelet
-            - pkg-cri-containerd
+      #- image-docker-weave:
+      #    requires:
+      #      - dependencies
+      #      - pkg-kubelet
+      #      - pkg-kubernetes-docker-image-cache-common
+      #      - pkg-kubernetes-docker-image-cache-control-plane
+      #- image-docker-bridge:
+      #    requires:
+      #      - dependencies
+      #      - pkg-kubelet
+      #      - pkg-kubernetes-docker-image-cache-common
+      #      - pkg-kubernetes-docker-image-cache-control-plane
+      #- image-cri-containerd-weave:
+      #    requires:
+      #      - dependencies
+      #      - pkg-kubelet
+      #      - pkg-cri-containerd
       - image-cri-containerd-bridge:
           requires:
             - dependencies
             - pkg-kubelet
             - pkg-cri-containerd
 
-      - boot-docker-weave:
-          requires:
-            - image-docker-weave
+      #- boot-docker-weave:
+      #    requires:
+      #      - image-docker-weave
 
       #- boot-docker-bridge:
       #    requires:
@@ -438,11 +438,11 @@ workflows:
             - pkg-kube-e2e-test
             - pkg-kubernetes-docker-image-cache-common
             - pkg-kubernetes-docker-image-cache-control-plane
-            - image-docker-weave
-            - image-docker-bridge
-            - image-cri-containerd-weave
+            #- image-docker-weave
+            #- image-docker-bridge
+            #- image-cri-containerd-weave
             - image-cri-containerd-bridge
-            - boot-docker-weave
+            #- boot-docker-weave
             #- boot-docker-bridge
             #- boot-cri-containerd-weave
             - boot-cri-containerd-bridge

From e39aebe83c8f9f74762a926bd8a078ac8869c9f9 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 13:08:50 +0000
Subject: [PATCH 08/14] Skip some more packages while debugging

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 566b36c..59b3306 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -257,13 +257,13 @@ jobs:
     <<: *linuxkit_pkg_build
   pkg-cri-containerd:
     <<: *linuxkit_pkg_build
-  pkg-kube-e2e-test:
-    <<: *linuxkit_pkg_build
+  #pkg-kube-e2e-test:
+  #  <<: *linuxkit_pkg_build
 
-  pkg-kubernetes-docker-image-cache-common:
-    <<: *linuxkit_pkg_build
-  pkg-kubernetes-docker-image-cache-control-plane:
-    <<: *linuxkit_pkg_build
+  #pkg-kubernetes-docker-image-cache-common:
+  #  <<: *linuxkit_pkg_build
+  #pkg-kubernetes-docker-image-cache-control-plane:
+  #  <<: *linuxkit_pkg_build
 
   #image-docker-weave:
   #  <<: *image_build
@@ -353,6 +353,8 @@ jobs:
                 exit 0
             fi
 
+            false # Debug branch, not expected to get this far
+            
             docker login -u $DOCKER_USER -p $DOCKER_PASS
             mkdir -p ~/.docker/trust/private
             cp .circleci/content-trust.key ~/.docker/trust/private/b056f84873aa0be205dfe826afa6e7458120c9569dd19a2a84154498fb1165d5.key
@@ -378,15 +380,15 @@ workflows:
       - pkg-cri-containerd:
           requires:
             - dependencies
-      - pkg-kube-e2e-test:
-          requires:
-            - dependencies
-      - pkg-kubernetes-docker-image-cache-common:
-          requires:
-            - dependencies
-      - pkg-kubernetes-docker-image-cache-control-plane:
-          requires:
-            - dependencies
+      #- pkg-kube-e2e-test:
+      #    requires:
+      #      - dependencies
+      #- pkg-kubernetes-docker-image-cache-common:
+      #    requires:
+      #      - dependencies
+      #- pkg-kubernetes-docker-image-cache-control-plane:
+      #    requires:
+      #      - dependencies
 
       #- image-docker-weave:
       #    requires:
@@ -435,9 +437,9 @@ workflows:
             - lint
             - pkg-kubelet
             - pkg-cri-containerd
-            - pkg-kube-e2e-test
-            - pkg-kubernetes-docker-image-cache-common
-            - pkg-kubernetes-docker-image-cache-control-plane
+            #- pkg-kube-e2e-test
+            #- pkg-kubernetes-docker-image-cache-common
+            #- pkg-kubernetes-docker-image-cache-control-plane
             #- image-docker-weave
             #- image-docker-bridge
             #- image-cri-containerd-weave

From 769d2234e17c5549375dc9f74ad62b324288b411 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 13:09:54 +0000
Subject: [PATCH 09/14] Move workspace to ~ for machine executor

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 59b3306..17e00cf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -96,13 +96,13 @@ image_boot: &image_boot
   steps:
     - run:
         name: Configure $PATH
-        command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
+        command: echo 'export PATH=~/workspace/bin:$PATH' >> $BASH_ENV
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
         command: sudo apt-get update && sudo apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso
     - attach_workspace:
-        at: /workspace
+        at: ~/workspace
     - checkout
     - run:
         name: Test boot
@@ -112,8 +112,8 @@ image_boot: &image_boot
           # TODO could this be integrated into linuxkit/mkimage-iso-*
           # such that all those options do not need to be repeated
           # here? e.g. perhaps this could be made to work:
-          # - Create tar with /root/.ssh/authorized_keys=/workspace/ssh/id_rsa.pub in it
-          # - cat $tar | docker run -v /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso:/input.iso linuxkit/mkimage-iso-bios -add-to /input.iso
+          # - Create tar with /root/.ssh/authorized_keys=~/workspace/ssh/id_rsa.pub in it
+          # - cat $tar | docker run -v ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso:/input.iso linuxkit/mkimage-iso-bios -add-to /input.iso
           #
           # Compared with the options used i nmkimage-iso-bios to
           # build the original dropped -J -joliet-long since xorriso
@@ -122,7 +122,7 @@ image_boot: &image_boot
           #
           #   Cannot add /bin/ls to Joliet tree. Symlinks can only be added to a Rock Ridge tree.
 
-          xorriso -indev /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso \
+          xorriso -indev ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso \
                   -outdev kube-master.iso \
                   -as genisoimage \
                     -l -R \
@@ -134,11 +134,11 @@ image_boot: &image_boot
                     -input-charset utf8 \
                   -- \
                   -pathspecs on \
-                  -add /root/.ssh/authorized_keys=/workspace/ssh/id_rsa.pub
+                  -add /root/.ssh/authorized_keys=~/workspace/ssh/id_rsa.pub
 
-          #cp /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-{master,node}.iso .
-          #cp /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso .
-          SSHOPTS="-i /workspace/ssh/id_rsa" ./test.exp
+          #cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-{master,node}.iso .
+          #cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso .
+          SSHOPTS="-i ~/workspace/ssh/id_rsa" ./test.exp
 
 version: 2
 jobs:

From 9b2ee0170ed7a96d9bfbe3462f649eef2a82f657 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 13:27:43 +0000
Subject: [PATCH 10/14] fix for shell ~ expansion and set -x

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 17e00cf..33d6cb0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -107,6 +107,7 @@ image_boot: &image_boot
     - run:
         name: Test boot
         command: |
+          set -x
           ls -lrt /dev/kvm* || true
           # Add ssh keys to the image.
           # TODO could this be integrated into linuxkit/mkimage-iso-*
@@ -122,6 +123,7 @@ image_boot: &image_boot
           #
           #   Cannot add /bin/ls to Joliet tree. Symlinks can only be added to a Rock Ridge tree.
 
+          id_rsa=~/workspace/ssh/id_rsa.pub
           xorriso -indev ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso \
                   -outdev kube-master.iso \
                   -as genisoimage \
@@ -134,7 +136,7 @@ image_boot: &image_boot
                     -input-charset utf8 \
                   -- \
                   -pathspecs on \
-                  -add /root/.ssh/authorized_keys=~/workspace/ssh/id_rsa.pub
+                  -add /root/.ssh/authorized_keys=$id_rsa
 
           #cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-{master,node}.iso .
           #cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso .

From 7a6924c53e2624c4ad7c6a2df314d13409710af5 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 13:57:11 +0000
Subject: [PATCH 11/14] Use less memory

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 2 ++
 test.exp             | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 33d6cb0..4481eed 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -108,6 +108,8 @@ image_boot: &image_boot
         name: Test boot
         command: |
           set -x
+          free
+          cat /proc/meminfo
           ls -lrt /dev/kvm* || true
           # Add ssh keys to the image.
           # TODO could this be integrated into linuxkit/mkimage-iso-*
diff --git a/test.exp b/test.exp
index 32eee0e..35b13c2 100755
--- a/test.exp
+++ b/test.exp
@@ -27,7 +27,7 @@ proc boot_linuxkit {} {
     # medium+             3.0     6GB
     # large               4.0     8GB     (7680M)
     # xlarge              8.0     16GB    (15827M)
-    spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=15872 KUBE_VCPUS=8 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y ./boot.sh
+    spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=3584 KUBE_VCPUS=8 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y ./boot.sh
     set lk_sid $spawn_id
     puts "INFO lk ($lk_sid) is pid [exp_pid -i $lk_sid]"
 }

From 1e91ab403607165829a82b2dfb9c52646bc920b6 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 14:25:46 +0000
Subject: [PATCH 12/14] more debug, less vcpus

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 boot.sh  | 2 +-
 test.exp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/boot.sh b/boot.sh
index a46b0a3..056c258 100755
--- a/boot.sh
+++ b/boot.sh
@@ -88,4 +88,4 @@ if [ -n "${kubeadm_data}" ] ; then
     echo "{  \"kubeadm\": { \"entries\": { ${kubeadm_data} } } }" > $state/metadata.json
 fi
 
-exec linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data $state/metadata.json ${uefi} "${img}${suffix}"
+exec linuxkit -v run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data $state/metadata.json ${uefi} "${img}${suffix}"
diff --git a/test.exp b/test.exp
index 35b13c2..21da2e1 100755
--- a/test.exp
+++ b/test.exp
@@ -27,7 +27,7 @@ proc boot_linuxkit {} {
     # medium+             3.0     6GB
     # large               4.0     8GB     (7680M)
     # xlarge              8.0     16GB    (15827M)
-    spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=3584 KUBE_VCPUS=8 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y ./boot.sh
+    spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=3584 KUBE_VCPUS=2 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y ./boot.sh
     set lk_sid $spawn_id
     puts "INFO lk ($lk_sid) is pid [exp_pid -i $lk_sid]"
 }

From 457e7d615b22a3d0b11a042aea69f4ef0d9ba26b Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 14:36:04 +0000
Subject: [PATCH 13/14] Workspace binaries in Linux subdir

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4481eed..134b496 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -4,7 +4,7 @@ linuxkit_pkg_build: &linuxkit_pkg_build
   steps:
     - run:
         name: Configure $PATH
-        command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
+        command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
@@ -38,7 +38,7 @@ image_build: &image_build
   steps:
     - run:
         name: Configure $PATH
-        command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
+        command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
@@ -96,7 +96,7 @@ image_boot: &image_boot
   steps:
     - run:
         name: Configure $PATH
-        command: echo 'export PATH=~/workspace/bin:$PATH' >> $BASH_ENV
+        command: echo 'export PATH=~/workspace/bin-Linux:$PATH' >> $BASH_ENV
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
@@ -152,7 +152,7 @@ jobs:
     steps:
       - run:
           name: Create workspace
-          command: mkdir -p /workspace/bin
+          command: mkdir -p /workspace/bin-Linux
       - run:
           name: Install packages
           command: apt-get update && apt-get install -y ca-certificates curl openssh-client
@@ -160,21 +160,21 @@ jobs:
           name: Fetch binaries
           command: |
             curl -fsSL -o /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz
-            tar xfO /tmp/docker.tgz docker/docker > /workspace/bin/docker
-            curl -fsSL -o /workspace/bin/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-linux-amd64
-            curl -fsSL -o /workspace/bin/manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64
-            curl -fsSL -o /workspace/bin/notary https://github.com/theupdateframework/notary/releases/download/v0.4.3/notary-Linux-amd64
+            tar xfO /tmp/docker.tgz docker/docker > /workspace/bin-Linux/docker
+            curl -fsSL -o /workspace/bin-Linux/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-linux-amd64
+            curl -fsSL -o /workspace/bin-Linux/manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64
+            curl -fsSL -o /workspace/bin-Linux/notary https://github.com/theupdateframework/notary/releases/download/v0.4.3/notary-Linux-amd64
 
             echo "Downloaded:"
-            sha256sum /workspace/bin/*
+            sha256sum /workspace/bin-Linux/*
             echo
 
             echo "Checking checksums"
             sha256sum -c <<EOF
-            6af40e74b2dbb2927882acab52d50bfc72551779d541957fc70b6adc325ee5ef  /workspace/bin/docker
-            b61188823c2491e9a1c16d275e41c68d5736f3f10577d01cb358261ff71bbad6  /workspace/bin/linuxkit
-            e4ca2ef0015a4be8597d31d9e3e70d88da33924ae72b0999e9f3b79304d4710d  /workspace/bin/manifest-tool
-            06cd02c4c2e7a3b1ad9899b03b3d4dde5392d964c675247d32f604a24661f839  /workspace/bin/notary
+            6af40e74b2dbb2927882acab52d50bfc72551779d541957fc70b6adc325ee5ef  /workspace/bin-Linux/docker
+            b61188823c2491e9a1c16d275e41c68d5736f3f10577d01cb358261ff71bbad6  /workspace/bin-Linux/linuxkit
+            e4ca2ef0015a4be8597d31d9e3e70d88da33924ae72b0999e9f3b79304d4710d  /workspace/bin-Linux/manifest-tool
+            06cd02c4c2e7a3b1ad9899b03b3d4dde5392d964c675247d32f604a24661f839  /workspace/bin-Linux/notary
             EOF
       - run:
           name: Generate SSH Keys
@@ -184,10 +184,10 @@ jobs:
       - run:
           name: Versions
           command: |
-             chmod +x /workspace/bin/docker # docker version deferred until daemon configured in relevant jobs
-             chmod +x /workspace/bin/linuxkit && /workspace/bin/linuxkit version
-             chmod +x /workspace/bin/manifest-tool && /workspace/bin/manifest-tool --version
-             chmod +x /workspace/bin/notary && /workspace/bin/notary version
+             chmod +x /workspace/bin-Linux/docker # docker version deferred until daemon configured in relevant jobs
+             chmod +x /workspace/bin-Linux/linuxkit && /workspace/bin-Linux/linuxkit version
+             chmod +x /workspace/bin-Linux/manifest-tool && /workspace/bin-Linux/manifest-tool --version
+             chmod +x /workspace/bin-Linux/notary && /workspace/bin-Linux/notary version
       - persist_to_workspace:
           root: /workspace
           paths: bin
@@ -201,7 +201,7 @@ jobs:
     steps:
       - run:
           name: Configure $PATH
-          command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
+          command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
       - run:
           name: Install packages
           # ca-certificates are needed for attach_workspace (and git over https)
@@ -325,7 +325,7 @@ jobs:
     steps:
       - run:
           name: Configure $PATH
-          command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
+          command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
       - run:
           name: Install packages
           # ca-certificates are needed for attach_workspace (and git over https)

From e3426510fe463c0f370b4762df30f208757d53c0 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 12 Jan 2018 14:49:08 +0000
Subject: [PATCH 14/14] Try booting with Hyperkit on MacOS

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 .circleci/config.yml | 46 ++++++++++++++++++++++++++++++--------------
 1 file changed, 32 insertions(+), 14 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 134b496..209ff36 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -90,27 +90,34 @@ image_build: &image_build
         paths: images
 
 image_boot: &image_boot
-  machine:
-    enabled: true
-    image: circleci/classic:latest # default Ubuntu 14.04 image
+  #machine:
+  #  enabled: true
+  #  image: circleci/classic:latest # default Ubuntu 14.04 image
+  macos:
+    xcode: "9.0"
   steps:
     - run:
         name: Configure $PATH
-        command: echo 'export PATH=~/workspace/bin-Linux:$PATH' >> $BASH_ENV
+        #command: echo 'export PATH=~/workspace/bin-Linux:$PATH' >> $BASH_ENV
+        command: echo 'export PATH=~/workspace/bin-Darwin:$PATH' >> $BASH_ENV
     - run:
         name: Install packages
         # ca-certificates are needed for attach_workspace (and git over https)
-        command: sudo apt-get update && sudo apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso
+        #command: sudo apt-get update && sudo apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso
+        command: brew install xorriso
     - attach_workspace:
         at: ~/workspace
     - checkout
+    - run:
+        name: Versions
+        command: |
+          set -x
+          ~/workspace/bin-Darwin/linuxkit version
+          ~/workspace/bin-Darwin/vpnkit version
     - run:
         name: Test boot
         command: |
           set -x
-          free
-          cat /proc/meminfo
-          ls -lrt /dev/kvm* || true
           # Add ssh keys to the image.
           # TODO could this be integrated into linuxkit/mkimage-iso-*
           # such that all those options do not need to be repeated
@@ -152,29 +159,38 @@ jobs:
     steps:
       - run:
           name: Create workspace
-          command: mkdir -p /workspace/bin-Linux
+          command: mkdir -p /workspace/bin-Linux /workspace/bin-Darwin
       - run:
           name: Install packages
           command: apt-get update && apt-get install -y ca-certificates curl openssh-client
       - run:
           name: Fetch binaries
           command: |
+            set -x
             curl -fsSL -o /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz
             tar xfO /tmp/docker.tgz docker/docker > /workspace/bin-Linux/docker
             curl -fsSL -o /workspace/bin-Linux/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-linux-amd64
+            curl -fsSL -o /workspace/bin-Darwin/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-darwin-amd64
             curl -fsSL -o /workspace/bin-Linux/manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64
             curl -fsSL -o /workspace/bin-Linux/notary https://github.com/theupdateframework/notary/releases/download/v0.4.3/notary-Linux-amd64
 
+            curl -fsSL -o /workspace/bin-Darwin/hyperkit https://452-55985023-gh.circle-artifacts.com/0/Users/distiller/hyperkit/build/hyperkit
+            curl -fsSL -o /tmp/vpnkit.tgz https://1066-58395340-gh.circle-artifacts.com/0/Users/distiller/vpnkit/vpnkit.tgz
+            tar xfO /tmp/vpnkit.tgz Contents/Resources/bin/vpnkit > /workspace/bin-Darwin/vpnkit
+
             echo "Downloaded:"
-            sha256sum /workspace/bin-Linux/*
+            sha256sum /workspace/bin-Linux/* /workspace/bin-Darwin/*
             echo
 
             echo "Checking checksums"
             sha256sum -c <<EOF
             6af40e74b2dbb2927882acab52d50bfc72551779d541957fc70b6adc325ee5ef  /workspace/bin-Linux/docker
             b61188823c2491e9a1c16d275e41c68d5736f3f10577d01cb358261ff71bbad6  /workspace/bin-Linux/linuxkit
+            908fae1c48122a245ddef8821a530a4c504f267ff1718b0c3688ef6f7aa3b65f  /workspace/bin-Darwin/linuxkit
             e4ca2ef0015a4be8597d31d9e3e70d88da33924ae72b0999e9f3b79304d4710d  /workspace/bin-Linux/manifest-tool
             06cd02c4c2e7a3b1ad9899b03b3d4dde5392d964c675247d32f604a24661f839  /workspace/bin-Linux/notary
+            298f650c9dd3ab8907b23f56cb21e33b794a5a8be163bbbcc7cf959112888b80  /workspace/bin-Darwin/hyperkit
+            72e34d9c0bc0a73bfb71f2b572849db5829ca41c3995245145f2e57c8ab40bd2  /workspace/bin-Darwin/vpnkit
             EOF
       - run:
           name: Generate SSH Keys
@@ -186,14 +202,16 @@ jobs:
           command: |
              chmod +x /workspace/bin-Linux/docker # docker version deferred until daemon configured in relevant jobs
              chmod +x /workspace/bin-Linux/linuxkit && /workspace/bin-Linux/linuxkit version
+             chmod +x /workspace/bin-Darwin/linuxkit # && /workspace/bin-Darwin/linuxkit version
              chmod +x /workspace/bin-Linux/manifest-tool && /workspace/bin-Linux/manifest-tool --version
              chmod +x /workspace/bin-Linux/notary && /workspace/bin-Linux/notary version
+             chmod +x /workspace/bin-Darwin/vpnkit # && /workspace/bin-Darwin/vpnkit version
       - persist_to_workspace:
           root: /workspace
-          paths: bin
-      - persist_to_workspace:
-          root: /workspace
-          paths: ssh
+          paths:
+            - bin-Linux
+            - bin-Darwin
+            - ssh
 
   lint:
     docker: