diff --git a/.architectures-lib b/.architectures-lib deleted file mode 100644 index 88e72f8bb..000000000 --- a/.architectures-lib +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -_awkArch() { - local awkExpr="$1"; shift - awk "$@" "/^#|^\$/ { next } $awkExpr" release-architectures -} - -hasBashbrewArch() { - local bashbrewArch="$1"; shift - _awkArch 'BEGIN { exitCode = 1 } $1 == bashbrewArch { exitCode = 0 } END { exit exitCode }' -v bashbrewArch="$bashbrewArch" -} - -apkArches() { - _awkArch '{ print $2 }' -} - -apkToDockerArch() { - local apkArch="$1"; shift - _awkArch '$2 == apkArch { print $3; exit }' -v apkArch="$apkArch" -} - -apkToBashbrewArch() { - local apkArch="$1"; shift - _awkArch '$2 == apkArch { print $1; exit }' -v apkArch="$apkArch" -} - -_generateParentRepoToArches() { - local repo="$1"; shift - local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' - - eval "declare -g -A parentRepoToArches=( $( - find -name 'Dockerfile' -exec awk ' - toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|.*\/.*)(:|$)/ { - print "'"$officialImagesUrl"'" $2 - } - ' '{}' + \ - | sort -u \ - | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' - ) )" -} -_generateParentRepoToArches 'docker' - -versionChannel() { - local version="$1"; shift # "17.06", "17.11-rc", etc - local rcVersion="${version%-rc}" - - local channel='stable' - if [ "$rcVersion" != "$version" ]; then - channel='test' - fi - - echo "$channel" -} diff --git a/.github/workflows/verify-templating.yml b/.github/workflows/verify-templating.yml new file mode 100644 index 000000000..7e833f1c7 --- /dev/null +++ b/.github/workflows/verify-templating.yml @@ -0,0 +1,22 @@ +name: Verify Templating + +on: + pull_request: + push: + +defaults: + run: + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + apply-templates: + name: Check For Uncomitted Changes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Apply Templates + run: ./apply-templates.sh + - name: Check Git Status + run: | + status="$(git status --short)" + [ -z "$status" ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..d548f66de --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.jq-template.awk diff --git a/19.03-rc/Dockerfile b/19.03-rc/Dockerfile index cd2d3fdaa..0d34e9289 100644 --- a/19.03-rc/Dockerfile +++ b/19.03-rc/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.12 RUN apk add --no-cache \ @@ -11,7 +17,6 @@ RUN apk add --no-cache \ # - docker run --rm debian:stretch grep '^hosts:' /etc/nsswitch.conf RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf -ENV DOCKER_CHANNEL test ENV DOCKER_VERSION 19.03.13-beta2 # TODO ENV DOCKER_SHA256 # https://github.com/docker/docker-ce/blob/5b073ee2cf564edee5adca05eee574142f7627bb/components/packaging/static/hash_files !! @@ -19,24 +24,24 @@ ENV DOCKER_VERSION 19.03.13-beta2 RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" apkArch="$(apk --print-arch)"; \ case "$apkArch" in \ -# amd64 - x86_64) dockerArch='x86_64' ;; \ -# arm32v6 - armhf) dockerArch='armel' ;; \ -# arm32v7 - armv7) dockerArch='armhf' ;; \ -# arm64v8 - aarch64) dockerArch='aarch64' ;; \ - *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;;\ + 'x86_64') \ + url='https://download.docker.com/linux/static/test/x86_64/docker-19.03.13-beta2.tgz'; \ + ;; \ + 'armhf') \ + url='https://download.docker.com/linux/static/test/armel/docker-19.03.13-beta2.tgz'; \ + ;; \ + 'armv7') \ + url='https://download.docker.com/linux/static/test/armhf/docker-19.03.13-beta2.tgz'; \ + ;; \ + 'aarch64') \ + url='https://download.docker.com/linux/static/test/aarch64/docker-19.03.13-beta2.tgz'; \ + ;; \ + *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ esac; \ \ - if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${dockerArch}/docker-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${dockerArch}'"; \ - exit 1; \ - fi; \ + wget -O docker.tgz "$url"; \ \ tar --extract \ --file docker.tgz \ diff --git a/19.03-rc/dind-rootless/Dockerfile b/19.03-rc/dind-rootless/Dockerfile index 7b974d084..352dcddd8 100644 --- a/19.03-rc/dind-rootless/Dockerfile +++ b/19.03-rc/dind-rootless/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM docker:19.03-rc-dind # busybox "ip" is insufficient: @@ -15,24 +21,15 @@ RUN set -eux; \ RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" apkArch="$(apk --print-arch)"; \ case "$apkArch" in \ -# amd64 - x86_64) dockerArch='x86_64' ;; \ -# arm32v6 - armhf) dockerArch='armel' ;; \ -# arm32v7 - armv7) dockerArch='armhf' ;; \ -# arm64v8 - aarch64) dockerArch='aarch64' ;; \ - *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;;\ + 'x86_64') \ + url='https://download.docker.com/linux/static/test/x86_64/docker-rootless-extras-19.03.13-beta2.tgz'; \ + ;; \ + *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ esac; \ \ - if ! wget -O rootless.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${dockerArch}/docker-rootless-extras-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-rootless-extras-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${dockerArch}'"; \ - exit 1; \ - fi; \ + wget -O rootless.tgz "$url"; \ \ tar --extract \ --file rootless.tgz \ diff --git a/19.03-rc/dind/Dockerfile b/19.03-rc/dind/Dockerfile index 563c07da6..b6c98a168 100644 --- a/19.03-rc/dind/Dockerfile +++ b/19.03-rc/dind/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM docker:19.03-rc # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies diff --git a/19.03-rc/git/Dockerfile b/19.03-rc/git/Dockerfile index 88183ef05..650367c63 100644 --- a/19.03-rc/git/Dockerfile +++ b/19.03-rc/git/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM docker:19.03-rc RUN apk add --no-cache git diff --git a/19.03/Dockerfile b/19.03/Dockerfile index 222ad589d..395b13a33 100644 --- a/19.03/Dockerfile +++ b/19.03/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.12 RUN apk add --no-cache \ @@ -11,7 +17,6 @@ RUN apk add --no-cache \ # - docker run --rm debian:stretch grep '^hosts:' /etc/nsswitch.conf RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf -ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 19.03.13 # TODO ENV DOCKER_SHA256 # https://github.com/docker/docker-ce/blob/5b073ee2cf564edee5adca05eee574142f7627bb/components/packaging/static/hash_files !! @@ -19,24 +24,24 @@ ENV DOCKER_VERSION 19.03.13 RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" apkArch="$(apk --print-arch)"; \ case "$apkArch" in \ -# amd64 - x86_64) dockerArch='x86_64' ;; \ -# arm32v6 - armhf) dockerArch='armel' ;; \ -# arm32v7 - armv7) dockerArch='armhf' ;; \ -# arm64v8 - aarch64) dockerArch='aarch64' ;; \ - *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;;\ + 'x86_64') \ + url='https://download.docker.com/linux/static/stable/x86_64/docker-19.03.13.tgz'; \ + ;; \ + 'armhf') \ + url='https://download.docker.com/linux/static/stable/armel/docker-19.03.13.tgz'; \ + ;; \ + 'armv7') \ + url='https://download.docker.com/linux/static/stable/armhf/docker-19.03.13.tgz'; \ + ;; \ + 'aarch64') \ + url='https://download.docker.com/linux/static/stable/aarch64/docker-19.03.13.tgz'; \ + ;; \ + *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ esac; \ \ - if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${dockerArch}/docker-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${dockerArch}'"; \ - exit 1; \ - fi; \ + wget -O docker.tgz "$url"; \ \ tar --extract \ --file docker.tgz \ diff --git a/19.03/dind-rootless/Dockerfile b/19.03/dind-rootless/Dockerfile index e9f8cfcd9..d0087314f 100644 --- a/19.03/dind-rootless/Dockerfile +++ b/19.03/dind-rootless/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM docker:19.03-dind # busybox "ip" is insufficient: @@ -15,24 +21,15 @@ RUN set -eux; \ RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" apkArch="$(apk --print-arch)"; \ case "$apkArch" in \ -# amd64 - x86_64) dockerArch='x86_64' ;; \ -# arm32v6 - armhf) dockerArch='armel' ;; \ -# arm32v7 - armv7) dockerArch='armhf' ;; \ -# arm64v8 - aarch64) dockerArch='aarch64' ;; \ - *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;;\ + 'x86_64') \ + url='https://download.docker.com/linux/static/stable/x86_64/docker-rootless-extras-19.03.13.tgz'; \ + ;; \ + *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ esac; \ \ - if ! wget -O rootless.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${dockerArch}/docker-rootless-extras-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-rootless-extras-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${dockerArch}'"; \ - exit 1; \ - fi; \ + wget -O rootless.tgz "$url"; \ \ tar --extract \ --file rootless.tgz \ diff --git a/19.03/dind/Dockerfile b/19.03/dind/Dockerfile index 37bcfa70f..aa10b267b 100644 --- a/19.03/dind/Dockerfile +++ b/19.03/dind/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM docker:19.03 # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies diff --git a/19.03/git/Dockerfile b/19.03/git/Dockerfile index 2c8edb9f8..5fa563b1e 100644 --- a/19.03/git/Dockerfile +++ b/19.03/git/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM docker:19.03 RUN apk add --no-cache git diff --git a/Dockerfile-dind-rootless.template b/Dockerfile-dind-rootless.template index 5d596cf02..043d192e0 100644 --- a/Dockerfile-dind-rootless.template +++ b/Dockerfile-dind-rootless.template @@ -1,4 +1,4 @@ -FROM docker:%%VERSION%%-dind +FROM docker:{{ env.version }}-dind # busybox "ip" is insufficient: # [rootlesskit:child ] error: executing [[ip tuntap add name tap0 mode tap] [ip link set tap0 address 02:50:00:00:00:01]]: exit status 1 @@ -15,13 +15,36 @@ RUN set -eux; \ RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" - %%ARCH-CASE%%; \ + apkArch="$(apk --print-arch)"; \ + case "$apkArch" in \ +{{ + [ + .arches | to_entries[] + | select(.value.rootlessExtrasUrl) + | .key as $bashbrewArch + | ( + { + amd64: "x86_64", + arm32v6: "armhf", + arm32v7: "armv7", + arm64v8: "aarch64", + } + | .[$bashbrewArch] // $bashbrewArch + ) as $apkArch + | .value + | ( +-}} + {{ $apkArch | @sh }}) \ + url={{ .rootlessExtrasUrl | @sh }}; \ + ;; \ +{{ + ) + ] | add +-}} + *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ + esac; \ \ - if ! wget -O rootless.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${dockerArch}/docker-rootless-extras-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-rootless-extras-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${dockerArch}'"; \ - exit 1; \ - fi; \ + wget -O rootless.tgz "$url"; \ \ tar --extract \ --file rootless.tgz \ diff --git a/Dockerfile-dind.template b/Dockerfile-dind.template index 04f2d31bc..d0a234da0 100644 --- a/Dockerfile-dind.template +++ b/Dockerfile-dind.template @@ -1,4 +1,4 @@ -FROM docker:%%VERSION%% +FROM docker:{{ env.version }} # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies RUN set -eux; \ @@ -31,7 +31,7 @@ RUN set -x \ && echo 'dockremap:165536:65536' >> /etc/subgid # https://github.com/docker/docker/tree/master/hack/dind -ENV DIND_COMMIT %%DIND-COMMIT%% +ENV DIND_COMMIT {{ .dindCommit }} RUN set -eux; \ wget -O /usr/local/bin/dind "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind"; \ diff --git a/Dockerfile-git.template b/Dockerfile-git.template index 1c5647123..0bf1a1ce4 100644 --- a/Dockerfile-git.template +++ b/Dockerfile-git.template @@ -1,3 +1,3 @@ -FROM docker:%%VERSION%% +FROM docker:{{ env.version }} RUN apk add --no-cache git diff --git a/Dockerfile.template b/Dockerfile.template index 002a4c708..a51f771a5 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -1,4 +1,4 @@ -FROM alpine:%%TAG%% +FROM alpine:{{ .alpine }} RUN apk add --no-cache \ ca-certificates \ @@ -11,21 +11,43 @@ RUN apk add --no-cache \ # - docker run --rm debian:stretch grep '^hosts:' /etc/nsswitch.conf RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf -ENV DOCKER_CHANNEL %%DOCKER-CHANNEL%% -ENV DOCKER_VERSION %%DOCKER-VERSION%% +ENV DOCKER_VERSION {{ .version }} # TODO ENV DOCKER_SHA256 # https://github.com/docker/docker-ce/blob/5b073ee2cf564edee5adca05eee574142f7627bb/components/packaging/static/hash_files !! # (no SHA file artifacts on download.docker.com yet as of 2017-06-07 though) RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" - %%ARCH-CASE%%; \ + apkArch="$(apk --print-arch)"; \ + case "$apkArch" in \ +{{ + [ + .arches | to_entries[] + | select(.value.dockerUrl) + | .key as $bashbrewArch + | ( + { + amd64: "x86_64", + arm32v6: "armhf", + arm32v7: "armv7", + arm64v8: "aarch64", + } + | .[$bashbrewArch] // $bashbrewArch + ) as $apkArch + | .value + | ( +-}} + {{ $apkArch | @sh }}) \ + url={{ .dockerUrl | @sh }}; \ + ;; \ +{{ + ) + ] | add +-}} + *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ + esac; \ \ - if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${dockerArch}/docker-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${dockerArch}'"; \ - exit 1; \ - fi; \ + wget -O docker.tgz "$url"; \ \ tar --extract \ --file docker.tgz \ diff --git a/apply-templates.sh b/apply-templates.sh new file mode 100755 index 000000000..65a4cbbf9 --- /dev/null +++ b/apply-templates.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +[ -f versions.json ] # run "versions.sh" first + +jqt='.jq-template.awk' +if [ -n "${BASHBREW_SCRIPTS:-}" ]; then + jqt="$BASHBREW_SCRIPTS/jq-template.awk" +elif [ "$BASH_SOURCE" -nt "$jqt" ]; then + wget -qO "$jqt" 'https://github.com/docker-library/bashbrew/raw/5f0c26381fb7cc78b2d217d58007800bdcfbcfa1/scripts/jq-template.awk' +fi + +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi + +generated_warning() { + cat <<-EOH + # + # NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" + # + # PLEASE DO NOT EDIT IT DIRECTLY. + # + + EOH +} + +for version; do + export version + + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" + + for variant in "${variants[@]}"; do + template="Dockerfile${variant:+-$variant}.template" + target="$version${variant:+/$variant}/Dockerfile" + + { + generated_warning + gawk -f "$jqt" "$template" + } > "$target" + done + + cp -a docker-entrypoint.sh modprobe.sh "$version/" + cp -a dockerd-entrypoint.sh "$version/dind/" +done diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 6b18cf4e8..24f0a69b1 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -1,44 +1,13 @@ #!/bin/bash set -eu -declare -A aliases=( - #[18.06]='edge' -) - -# used for auto-detecting the "latest" of each channel (for tagging it appropriately) -# https://blog.docker.com/2017/03/docker-enterprise-edition/ -declare -A latestChannelRelease=() - self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -source '.architectures-lib' - -parentArches() { - local version="$1"; shift # "17.06", etc - - local parent="$(awk 'toupper($1) == "FROM" { print $2 }' "$version/Dockerfile")" - echo "${parentRepoToArches[$parent]:-}" -} -versionArches() { - local version="$1"; shift - - local parentArches="$(parentArches "$version")" - - local versionArches=() - for arch in $parentArches; do - if hasBashbrewArch "$arch" && grep -qE "^# $arch\$" "$version/Dockerfile"; then - versionArches+=( "$arch" ) - fi - done - echo "${versionArches[*]}" -} - -versions=( */ ) -versions=( "${versions[@]%/}" ) - -# sort version numbers with highest first -IFS=$'\n'; versions=( $(echo "${versions[*]}" | sort -rV) ); unset IFS +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi # get the most recent commit which modified any of "$@" fileCommit() { @@ -62,6 +31,48 @@ dirCommit() { ) } +getArches() { + local repo="$1"; shift + local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' + + eval "declare -A -g parentRepoToArches=( $( + find -name 'Dockerfile' -exec awk ' + toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|.*\/.*)(:|$)/ { + print "'"$officialImagesUrl"'" $2 + } + ' '{}' + \ + | sort -u \ + | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' + ) )" +} +getArches 'docker' + +versionArches() { + local version="$1"; shift + local variant="${1:-}" + local selector='dockerUrl' + if [[ "$variant" = *rootless ]]; then + selector='rootlessExtrasUrl' + fi + + local parent parentArches + parent="$(awk 'toupper($1) == "FROM" { print $2 }' "$version/Dockerfile")" + parentArches="${parentRepoToArches[$parent]:-}" + + comm -12 \ + <( + version="$version" jq -r ' + .[env.version].arches | to_entries[] + | select(.value.'"$selector"') + | .key + ' versions.json | sort + ) \ + <(xargs -n1 <<<"$parentArches" | sort) +} + +# sort version numbers with highest first +IFS=$'\n'; set -- $(sort -rV <<<"$*"); unset IFS + cat <<-EOH # this file is generated via https://github.com/docker-library/docker/blob/$(fileCommit "$self")/$self @@ -77,16 +88,20 @@ join() { echo "${out#$sep}" } -for version in "${versions[@]}"; do - rcVersion="${version%-rc}" +# used for auto-detecting the "latest" of each channel (for tagging it appropriately) +# https://blog.docker.com/2017/03/docker-enterprise-edition/ +declare -A latestChannelRelease=() - commit="$(dirCommit "$version")" +for version; do + export version + rcVersion="${version%-rc}" - fullVersion="$(git show "$commit":"$version/Dockerfile" | awk '$1 == "ENV" && $2 == "DOCKER_VERSION" { print $3; exit }')" + fullVersion="$(jq -r '.[env.version].version' versions.json)" if [ "$rcVersion" != "$version" ] && [ -e "$rcVersion/Dockerfile" ]; then # if this is a "-rc" release, let's make sure the release it contains isn't already GA (and thus something we should not publish anymore) - rcFullVersion="$(git show HEAD:"$rcVersion/Dockerfile" | awk '$1 == "ENV" && $2 == "DOCKER_VERSION" { print $3; exit }')" + export rcVersion + rcFullVersion="$(jq -r '.[env.rcVersion].version' versions.json)" latestVersion="$({ echo "$fullVersion"; echo "$rcFullVersion"; } | sort -V | tail -1)" if [[ "$fullVersion" == "$rcFullVersion"* ]] || [ "$latestVersion" = "$rcFullVersion" ]; then # "x.y.z-rc1" == x.y.z* @@ -108,7 +123,6 @@ for version in "${versions[@]}"; do ) # add a few channel/version-related aliases - channel="$(versionChannel "$version")" majorVersion="${version%%.*}" if [ "$version" != "$rcVersion" ] && [ -z "${latestChannelRelease['rc']:-}" ]; then versionAliases+=( 'rc' ) @@ -118,9 +132,11 @@ for version in "${versions[@]}"; do versionAliases+=( "$majorVersion" ) latestChannelRelease["$majorVersion"]="$version" fi - versionAliases+=( - ${aliases[$version]:-} - ) + + channel='stable' + if [ "$rcVersion" != "$version" ]; then + channel='test' + fi if [ -z "${latestChannelRelease[$channel]:-}" ]; then if [ "$rcVersion" = '19.03' ]; then versionAliases+=( "$channel" ); fi # 19.03 is the last release to include any "channel" aliases latestChannelRelease[$channel]="$version" @@ -135,54 +151,24 @@ for version in "${versions[@]}"; do latestChannelRelease['latest']="$version" fi - versionArches="$(versionArches "$version")" - - echo - cat <<-EOE - Tags: $(join ', ' "${versionAliases[@]}") - Architectures: $(join ', ' $versionArches) - GitCommit: $commit - Directory: $version - EOE - - for v in \ - dind dind-rootless git \ - windows/windowsservercore-{ltsc2016,1709} \ - ; do - dir="$version/$v" + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" + + for variant in "${variants[@]}"; do + dir="$version${variant:+/$variant}" [ -f "$dir/Dockerfile" ] || continue - variant="$(basename "$v")" commit="$(dirCommit "$dir")" - variantAliases=( "${versionAliases[@]/%/-$variant}" ) + variantAliases=( "${versionAliases[@]/%/${variant:+-$variant}}" ) variantAliases=( "${variantAliases[@]//latest-/}" ) - case "$v" in - # https://github.com/docker/docker-ce/blob/8fb3bb7b2210789a4471c017561c1b0de0b4f145/components/engine/hack/make/binary-daemon#L24 - # "vpnkit is amd64-only" ... for now?? - dind-rootless) variantArches='amd64' ;; - - windows/*) variantArches='windows-amd64' ;; - *) variantArches="$versionArches" ;; - esac - - sharedTags=() - if [[ "$variant" == 'windowsservercore'* ]]; then - sharedTags=( "${versionAliases[@]/%/-windowsservercore}" ) - sharedTags=( "${sharedTags[@]//latest-/}" ) - fi - echo - echo "Tags: $(join ', ' "${variantAliases[@]}")" - if [ "${#sharedTags[@]}" -gt 0 ]; then - echo "SharedTags: $(join ', ' "${sharedTags[@]}")" - fi cat <<-EOE - Architectures: $(join ', ' $variantArches) + Tags: $(join ', ' "${variantAliases[@]}") + Architectures: $(join ', ' $(versionArches "$version" "$variant")) GitCommit: $commit Directory: $dir EOE - [ "$variant" = "$v" ] || echo "Constraints: $variant" done done diff --git a/release-architectures b/release-architectures deleted file mode 100644 index 1271c0eb9..000000000 --- a/release-architectures +++ /dev/null @@ -1,9 +0,0 @@ -# see subdirectories of https://download.docker.com/linux/static/ - -# bashbrew-arch apk-arch docker-release-arch -amd64 x86_64 x86_64 -arm32v6 armhf armel -arm32v7 armv7 armhf -arm64v8 aarch64 aarch64 -ppc64le ppc64le ppc64le -s390x s390x s390x diff --git a/update.sh b/update.sh index 1b339f4d0..f19a544c0 100755 --- a/update.sh +++ b/update.sh @@ -1,141 +1,7 @@ #!/bin/bash set -eo pipefail -defaultAlpineVersion='3.12' -declare -A alpineVersion=( - #[17.09]='3.6' -) - cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -source '.architectures-lib' - -versions=( "$@" ) -if [ ${#versions[@]} -eq 0 ]; then - versions=( */ ) -fi -versions=( "${versions[@]%/}" ) - -# see http://stackoverflow.com/a/2705678/433558 -sed_escape_lhs() { - echo "$@" | sed -e 's/[]\/$*.^|[]/\\&/g' -} -sed_escape_rhs() { - echo "$@" | sed -e 's/[\/&]/\\&/g' | sed -e ':a;N;$!ba;s/\n/\\n/g' -} - -# "tac|tac" for http://stackoverflow.com/a/28879552/433558 -dindLatest="$(curl -fsSL 'https://github.com/docker/docker/commits/master/hack/dind.atom' | tac|tac | awk -F '[[:space:]]*[<>/]+' '$2 == "id" && $3 ~ /Commit/ { print $4; exit }')" - -dockerVersions="$( - git ls-remote --tags https://github.com/docker/docker-ce.git \ - | cut -d$'\t' -f2 \ - | grep '^refs/tags/v[0-9].*$' \ - | sed 's!^refs/tags/v!!; s!\^{}$!!' \ - | sort -u \ - | gawk ' - { data[lines++] = $0 } - - # "beta" sorts lower than "tp" even though "beta" is a more preferred release, so we need to explicitly adjust the sorting order for RCs - # also, "18.09.0-ce-beta1" vs "18.09.0-beta3" - function docker_version_compare(i1, v1, i2, v2, l, r) { - l = v1; gsub(/-ce/, "", l); gsub(/-tp/, "-alpha", l) - r = v2; gsub(/-ce/, "", r); gsub(/-tp/, "-alpha", r) - patsplit(l, ltemp, /[^.-]+/) - patsplit(r, rtemp, /[^.-]+/) - for (i = 0; i < length(ltemp) && i < length(rtemp); ++i) { - if (ltemp[i] < rtemp[i]) { - return -1 - } - if (ltemp[i] > rtemp[i]) { - return 1 - } - } - return 0 - } - - END { - asort(data, result, "docker_version_compare") - for (i in result) { - print result[i] - } - } - ' -)" - -for version in "${versions[@]}"; do - rcVersion="${version%-rc}" - - versionOptions="$(grep "^$rcVersion[.]" <<<"$dockerVersions")" - - rcGrepV='-v' - if [ "$rcVersion" != "$version" ]; then - rcGrepV= - fi - - fullVersion="$(grep $rcGrepV -E -- '-(rc|tp|beta)' <<<"$versionOptions" | tail -1)" - if [ -z "$fullVersion" ]; then - echo >&2 "warning: cannot find full version for $version" - continue - fi - - channel="$(versionChannel "$version")" - - echo "$version: $fullVersion ($channel)" - - archCase='apkArch="$(apk --print-arch)"; '$'\\\n' - archCase+=$'\t''case "$apkArch" in '$'\\\n' - for apkArch in $(apkArches); do - dockerArch="$(apkToDockerArch "$apkArch")" - # check whether the given architecture is supported for this release - if wget --quiet --spider "https://download.docker.com/linux/static/$channel/$dockerArch/docker-$fullVersion.tgz" &> /dev/null; then - bashbrewArch="$(apkToBashbrewArch "$apkArch")" - archCase+="# $bashbrewArch"$'\n' - archCase+=$'\t\t'"$apkArch) dockerArch='$dockerArch' ;; "$'\\\n' - fi - done - archCase+=$'\t\t''*) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;;'$'\\\n' - archCase+=$'\t''esac' - - alpine="${alpineVersion[$version]:-$defaultAlpineVersion}" - - majorVersion="${fullVersion%%.*}" - minorVersion="${fullVersion#$majorVersion.}" - minorVersion="${minorVersion%%.*}" - minorVersion="${minorVersion#0}" - - for variant in \ - '' git dind dind-rootless \ - windows/windowsservercore-{1709,ltsc2016} \ - ; do - dir="$version${variant:+/$variant}" - [ -d "$dir" ] || continue - df="$dir/Dockerfile" - slash='/' - case "$variant" in - windows/windowsservercore*) tag="${variant#*-}"; template='Dockerfile-windows-windowsservercore.template' ;; - *) tag="$alpine"; template="Dockerfile${variant:+-${variant//$slash/-}}.template" ;; - esac - sed -r \ - -e 's!%%VERSION%%!'"$version"'!g' \ - -e 's!%%DOCKER-CHANNEL%%!'"$channel"'!g' \ - -e 's!%%DOCKER-VERSION%%!'"$fullVersion"'!g' \ - -e 's!%%TAG%%!'"$tag"'!g' \ - -e 's!%%DIND-COMMIT%%!'"$dindLatest"'!g' \ - -e 's!%%ARCH-CASE%%!'"$(sed_escape_rhs "$archCase")"'!g' \ - "$template" > "$df" - - # DOCKER_TLS_CERTDIR is only enabled-by-default in 19.03+ - if [ "$majorVersion" -lt 19 ]; then - sed -ri -e 's!^(ENV DOCKER_TLS_CERTDIR=).*$!\1!' "$df" - fi - - # pigz (https://github.com/moby/moby/pull/35697) is only 18.02+ - if [ "$majorVersion" -lt 18 ] || { [ "$majorVersion" -eq 18 ] && [ "$minorVersion" -lt 2 ]; }; then - sed -ri '/pigz/d' "$df" - fi - done - - cp -a docker-entrypoint.sh modprobe.sh "$version/" - cp -a dockerd-entrypoint.sh "$version/dind/" -done +./versions.sh "$@" +./apply-templates.sh "$@" diff --git a/versions.json b/versions.json new file mode 100644 index 000000000..336eed69b --- /dev/null +++ b/versions.json @@ -0,0 +1,54 @@ +{ + "19.03": { + "alpine": "3.12", + "arches": { + "amd64": { + "dockerUrl": "https://download.docker.com/linux/static/stable/x86_64/docker-19.03.13.tgz", + "rootlessExtrasUrl": "https://download.docker.com/linux/static/stable/x86_64/docker-rootless-extras-19.03.13.tgz" + }, + "arm32v6": { + "dockerUrl": "https://download.docker.com/linux/static/stable/armel/docker-19.03.13.tgz" + }, + "arm32v7": { + "dockerUrl": "https://download.docker.com/linux/static/stable/armhf/docker-19.03.13.tgz" + }, + "arm64v8": { + "dockerUrl": "https://download.docker.com/linux/static/stable/aarch64/docker-19.03.13.tgz" + } + }, + "dindCommit": "ed89041433a031cafc0a0f19cfe573c31688d377", + "variants": [ + "", + "dind", + "dind-rootless", + "git" + ], + "version": "19.03.13" + }, + "19.03-rc": { + "alpine": "3.12", + "arches": { + "amd64": { + "dockerUrl": "https://download.docker.com/linux/static/test/x86_64/docker-19.03.13-beta2.tgz", + "rootlessExtrasUrl": "https://download.docker.com/linux/static/test/x86_64/docker-rootless-extras-19.03.13-beta2.tgz" + }, + "arm32v6": { + "dockerUrl": "https://download.docker.com/linux/static/test/armel/docker-19.03.13-beta2.tgz" + }, + "arm32v7": { + "dockerUrl": "https://download.docker.com/linux/static/test/armhf/docker-19.03.13-beta2.tgz" + }, + "arm64v8": { + "dockerUrl": "https://download.docker.com/linux/static/test/aarch64/docker-19.03.13-beta2.tgz" + } + }, + "dindCommit": "ed89041433a031cafc0a0f19cfe573c31688d377", + "variants": [ + "", + "dind", + "dind-rootless", + "git" + ], + "version": "19.03.13-beta2" + } +} diff --git a/versions.sh b/versions.sh new file mode 100755 index 000000000..2f7fd43a7 --- /dev/null +++ b/versions.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +defaultAlpineVersion='3.12' +declare -A alpineVersion=( + #[17.09]='3.6' +) + +# bashbrew arch to docker-release-arch +declare -A dockerArches=( + ['amd64']='x86_64' + ['arm32v6']='armel' + ['arm32v7']='armhf' + ['arm64v8']='aarch64' + ['ppc64le']='ppc64le' + ['s390x']='s390x' +# ['windows-amd64']='windows-amd64' +) + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" + +versions=( "$@" ) +if [ ${#versions[@]} -eq 0 ]; then + versions=( */ ) + json='{}' +else + json="$(< versions.json)" +fi +versions=( "${versions[@]%/}" ) + +# "tac|tac" for http://stackoverflow.com/a/28879552/433558 +dindLatest="$(curl -fsSL 'https://github.com/docker/docker/commits/master/hack/dind.atom' | tac|tac | awk -F '[[:space:]]*[<>/]+' '$2 == "id" && $3 ~ /Commit/ { print $4; exit }')" + +dockerVersions="$( + git ls-remote --tags https://github.com/docker/docker-ce.git \ + | cut -d$'\t' -f2 \ + | grep '^refs/tags/v[0-9].*$' \ + | sed 's!^refs/tags/v!!; s!\^{}$!!' \ + | sort -u \ + | gawk ' + { data[lines++] = $0 } + + # "beta" sorts lower than "tp" even though "beta" is a more preferred release, so we need to explicitly adjust the sorting order for RCs + # also, "18.09.0-ce-beta1" vs "18.09.0-beta3" + function docker_version_compare(i1, v1, i2, v2, l, r) { + l = v1; gsub(/-ce/, "", l); gsub(/-tp/, "-alpha", l) + r = v2; gsub(/-ce/, "", r); gsub(/-tp/, "-alpha", r) + patsplit(l, ltemp, /[^.-]+/) + patsplit(r, rtemp, /[^.-]+/) + for (i = 0; i < length(ltemp) && i < length(rtemp); ++i) { + if (ltemp[i] < rtemp[i]) { + return -1 + } + if (ltemp[i] > rtemp[i]) { + return 1 + } + } + return 0 + } + + END { + asort(data, result, "docker_version_compare") + for (i in result) { + print result[i] + } + } + ' +)" + +for version in "${versions[@]}"; do + rcVersion="${version%-rc}" + channel='stable' + alpine="${alpineVersion[$version]:-$defaultAlpineVersion}" + + versionOptions="$(grep "^$rcVersion[.]" <<<"$dockerVersions")" + + rcGrepV='-v' + if [ "$rcVersion" != "$version" ]; then + rcGrepV= + channel='test' + fi + + fullVersion="$(grep $rcGrepV -E -- '-(rc|tp|beta)' <<<"$versionOptions" | tail -1)" + if [ -z "$fullVersion" ]; then + echo >&2 "warning: cannot find full version for $version" + continue + fi + + echo "$version: $fullVersion" + + export fullVersion alpine dindLatest + doc="$( + jq -nc '{ + version: env.fullVersion, + arches: {}, + alpine: env.alpine, + dindCommit: env.dindLatest, + }' + )" + + for bashbrewArch in "${!dockerArches[@]}"; do + arch="${dockerArches[$bashbrewArch]}" + # check whether the given architecture is supported for this release + url="https://download.docker.com/linux/static/$channel/$arch/docker-$fullVersion.tgz" + if wget --quiet --spider "$url" &> /dev/null; then + export bashbrewArch url + doc="$( + jq <<<"$doc" -c '.arches[env.bashbrewArch] = { + dockerUrl: env.url, + }' + )" + else + continue + fi + + rootlessExtrasUrl="https://download.docker.com/linux/static/$channel/$arch/docker-rootless-extras-$fullVersion.tgz" + # https://github.com/docker/docker-ce/blob/8fb3bb7b2210789a4471c017561c1b0de0b4f145/components/engine/hack/make/binary-daemon#L24 + # "vpnkit is amd64-only" ... for now?? + if [ "$bashbrewArch" = 'amd64' ] && wget --quiet --spider "$rootlessExtrasUrl" &> /dev/null; then + export rootlessExtrasUrl + doc="$( + jq <<<"$doc" -c \ + '.arches[env.bashbrewArch].rootlessExtrasUrl = env.rootlessExtrasUrl' + )" + fi + done + + # order here controls the order of the library/ file + for variant in \ + '' \ + dind \ + dind-rootless \ + git \ + ; do + base="${variant%%/*}" # "buster", "windows", etc. + [ -d "$version/$base" ] || continue + export variant + doc="$(jq <<<"$doc" -c '.variants += [ env.variant ]')" + done + + export version + json="$(jq <<<"$json" -c --argjson doc "$doc" '.[env.version] = $doc')" +done + +jq <<<"$json" -S . > versions.json