From 99dafa4279156d3064da280c6d3b2f95a788d74d Mon Sep 17 00:00:00 2001 From: Konstantinos Smanis Date: Sat, 27 Jul 2024 12:27:58 +0300 Subject: fix: reduce Docker registry pull requests Circumvent the Docker Hub rate limit triggered by pushing the multiarch manifests after *every* image is built and pushed. Instead, deploy the manifests only after *all* images have been built and pushed. Signed-off-by: Konstantinos Smanis Signed-off-by: John Helmert III --- .github/actions/container_build/action.yml | 2 - .github/actions/manifest_build/action.yml | 25 +++++++++++++ .github/workflows/build.yml | 30 +++++++++++++++ deploy-manifests.sh | 59 ++++++++++++++++++++++++++++++ deploy.sh | 54 --------------------------- 5 files changed, 114 insertions(+), 56 deletions(-) create mode 100644 .github/actions/manifest_build/action.yml create mode 100755 deploy-manifests.sh diff --git a/.github/actions/container_build/action.yml b/.github/actions/container_build/action.yml index fc2b9b9..56584ee 100644 --- a/.github/actions/container_build/action.yml +++ b/.github/actions/container_build/action.yml @@ -45,6 +45,4 @@ runs: github.ref_name == 'master' && github.repository_owner == 'gentoo' && (github.event_name == 'schedule' || github.event_name == 'push') shell: bash - env: - DOCKER_CLI_EXPERIMENTAL: enabled run: ./deploy.sh diff --git a/.github/actions/manifest_build/action.yml b/.github/actions/manifest_build/action.yml new file mode 100644 index 0000000..e1b7829 --- /dev/null +++ b/.github/actions/manifest_build/action.yml @@ -0,0 +1,25 @@ +--- +name: Build manifest +inputs: + target: + required: true + dockerhub_username: + required: true + dockerhub_password: + required: true +runs: + using: composite + steps: + - name: Set up TARGET + run: echo "TARGET=${{ inputs.target }}" | tee $GITHUB_ENV + shell: bash + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ inputs.dockerhub_username }} + password: ${{ inputs.dockerhub_password }} + - name: Push manifests + shell: bash + env: + DOCKER_CLI_EXPERIMENTAL: enabled + run: ./deploy-manifests.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0e22e8..64d0c37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,7 @@ env: jobs: build: + continue-on-error: true strategy: fail-fast: false matrix: @@ -67,3 +68,32 @@ jobs: target: ${{ matrix.target }} dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + build-manifests: + if: | + github.ref_name == 'master' && github.repository_owner == 'gentoo' && + (github.event_name == 'schedule' || github.event_name == 'push') + needs: [build] + strategy: + fail-fast: false + matrix: + target: + - stage3:latest + - stage3:desktop + - stage3:hardened + - stage3:hardened-nomultilib + - stage3:musl + - stage3:musl-hardened + - stage3:nomultilib + - stage3:nomultilib-systemd + - stage3:systemd + name: ${{ matrix.target }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Build ${{ matrix.target }} + uses: ./.github/actions/manifest_build + with: + target: ${{ matrix.target }} + dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/deploy-manifests.sh b/deploy-manifests.sh new file mode 100755 index 0000000..08bc158 --- /dev/null +++ b/deploy-manifests.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +if [[ -z "$TARGET" ]]; then + echo "TARGET environment variable must be set e.g. TARGET=stage3:latest." + exit 1 +fi + +# Split the TARGET variable into two elements separated by colons +IFS=: read -r NAME MANIFEST_TAG <<< "${TARGET}" + +VERSION=${VERSION:-$(date -u +%Y%m%d)} + +ORG=${ORG:-gentoo} + +declare -A MANIFEST_TAGS=( + [stage3:latest]="amd64-openrc;armv5tel-openrc;armv6j_hardfp-openrc;armv7a_hardfp-openrc;arm64-openrc;i686-openrc;ppc64le-openrc;rv64_lp64d-openrc;s390x" + [stage3:desktop]="amd64-desktop-openrc;arm64-desktop-openrc" + [stage3:hardened]="amd64-hardened-openrc;i686-hardened-openrc" + [stage3:hardened-nomultilib]="amd64-hardened-nomultilib-openrc" + [stage3:musl]="amd64-musl;armv7a_hardfp_musl-openrc;arm64-musl;i686-musl" + [stage3:musl-hardened]="amd64-musl-hardened;arm64-musl-hardened;ppc64le-musl-hardened-openrc" + [stage3:nomultilib]="amd64-nomultilib-openrc" + [stage3:nomultilib-systemd]="amd64-nomultilib-systemd" + [stage3:systemd]="amd64-systemd;armv5tel-systemd;armv6j_hardfp-systemd;armv7a_hardfp-systemd;arm64-systemd;i686-systemd;ppc64le-systemd;rv64_lp64d-systemd" +) + +IFS=';' read -ra TAGS <<< "${MANIFEST_TAGS[${TARGET}]}" +if [[ -z "${TAGS+x}" ]]; then + echo "Done! No manifests to push for TARGET=${TARGET}." + exit 0 +fi +MANIFEST="${TARGET}" + +# Latest manifests +IMAGES=() +for TAG in "${TAGS[@]}"; do + IMAGE="${ORG}/${NAME}:${TAG}" + if docker manifest inspect "${IMAGE}" &>/dev/null; then + IMAGES+=("${IMAGE}") + fi +done + +docker manifest create "${ORG}/${MANIFEST}" "${IMAGES[@]}" +docker manifest push "${ORG}/${MANIFEST}" + +# Dated manifests +MANIFEST="${MANIFEST}-${VERSION}" +MANIFEST="${MANIFEST/:latest-/:}" # Remove "latest" tag prefix + +IMAGES=() +for TAG in "${TAGS[@]}"; do + IMAGE="${ORG}/${NAME}:${TAG}-${VERSION}" + if docker manifest inspect "${IMAGE}" &>/dev/null; then + IMAGES+=("${IMAGE}") + fi +done + +docker manifest create "${ORG}/${MANIFEST}" "${IMAGES[@]}" +docker manifest push "${ORG}/${MANIFEST}" diff --git a/deploy.sh b/deploy.sh index 71d5d90..4871735 100755 --- a/deploy.sh +++ b/deploy.sh @@ -8,61 +8,7 @@ fi # Split the TARGET variable into three elements separated by hyphens IFS=- read -r NAME ARCH SUFFIX <<< "${TARGET}" -VERSION=${VERSION:-$(date -u +%Y%m%d)} - ORG=${ORG:-gentoo} # Push built images docker push --all-tags "${ORG}/${NAME}" - -declare -A MANIFEST_TAGS=( - [stage3:latest]="amd64-openrc;armv5tel-openrc;armv6j_hardfp-openrc;armv7a_hardfp-openrc;arm64-openrc;i686-openrc;ppc64le-openrc;rv64_lp64d-openrc;s390x" - [stage3:desktop]="amd64-desktop-openrc;arm64-desktop-openrc" - [stage3:hardened]="amd64-hardened-openrc;i686-hardened-openrc" - [stage3:hardened-nomultilib]="amd64-hardened-nomultilib-openrc" - [stage3:musl]="amd64-musl;armv7a_hardfp_musl-openrc;arm64-musl;i686-musl" - [stage3:musl-hardened]="amd64-musl-hardened;arm64-musl-hardened;ppc64le-musl-hardened-openrc" - [stage3:nomultilib]="amd64-nomultilib-openrc" - [stage3:nomultilib-systemd]="amd64-nomultilib-systemd" - [stage3:systemd]="amd64-systemd;armv5tel-systemd;armv6j_hardfp-systemd;armv7a_hardfp-systemd;arm64-systemd;i686-systemd;ppc64le-systemd;rv64_lp64d-systemd" -) - -# Find latest manifest -TAG="${ARCH}${SUFFIX:+-${SUFFIX}}" -for MANIFEST in "${!MANIFEST_TAGS[@]}"; do - if [[ "${MANIFEST_TAGS[${MANIFEST}]}" =~ (^|;)"${TAG}"(;|$) ]]; then - IFS=';' read -ra TAGS <<< "${MANIFEST_TAGS[${MANIFEST}]}" - break - fi -done -if [[ -z "${TAGS+x}" ]]; then - echo "Done! No manifests to push for TARGET=${TARGET}." - exit 0 -fi - -# Latest manifests -IMAGES=() -for TAG in "${TAGS[@]}"; do - IMAGE="${ORG}/${NAME}:${TAG}" - if docker manifest inspect "${IMAGE}" &>/dev/null; then - IMAGES+=("${IMAGE}") - fi -done - -docker manifest create "${ORG}/${MANIFEST}" "${IMAGES[@]}" -docker manifest push "${ORG}/${MANIFEST}" - -# Dated manifests -MANIFEST="${MANIFEST}-${VERSION}" -MANIFEST="${MANIFEST/:latest-/:}" # Remove "latest" tag prefix - -IMAGES=() -for TAG in "${TAGS[@]}"; do - IMAGE="${ORG}/${NAME}:${TAG}-${VERSION}" - if docker manifest inspect "${IMAGE}" &>/dev/null; then - IMAGES+=("${IMAGE}") - fi -done - -docker manifest create "${ORG}/${MANIFEST}" "${IMAGES[@]}" -docker manifest push "${ORG}/${MANIFEST}" -- cgit v1.2.3-65-gdbad