diff options
| -rwxr-xr-x | create-squashfs-snapshot | 104 | ||||
| -rwxr-xr-x | rsync-gen.sh | 12 | ||||
| -rwxr-xr-x | sync-origin-mirror-releases.sh | 15 |
3 files changed, 117 insertions, 14 deletions
diff --git a/create-squashfs-snapshot b/create-squashfs-snapshot index 4fc35ff..2095ebf 100755 --- a/create-squashfs-snapshot +++ b/create-squashfs-snapshot @@ -26,6 +26,7 @@ shopt -s nullglob set -e -x +date +%s.%N # == config == # filled with gentoo-specific details, change at will @@ -74,7 +75,7 @@ trap 'rm -rf "${tempdir}"' SIGINT SIGTERM EXIT # Build exclusion list EXCLUSION_LIST="$(mktemp -p "${tempdir}" squashfs-exclude.XXXXXXXXXX)" -"$(dirname $0)"/print-exclusion-list.sh "${repodir}" >"${EXCLUSION_LIST}" +"$(dirname "$0")"/print-exclusion-list.sh "${repodir}" >"${EXCLUSION_LIST}" mksquashfs_options+=( -ef "${EXCLUSION_LIST}" ) @@ -100,7 +101,7 @@ for algo in "${algo_LIST[@]}" ; do if [[ ! -f ${todaysnap} ]]; then # take today's snapshot - tmp=algo_$algo[@] + tmp="algo_${algo}[@]" file="${tempdir}/${reponame}-${today}${ext}" mksquashfs "${repodir}" "${file}" "${mksquashfs_options[@]}" "${!tmp}" mv "${file}" "${mirrordir}/" @@ -117,7 +118,7 @@ for algo in "${algo_LIST[@]}" ; do revdeltas=( "${revdeltadir}"/*.${algo}.sqdelta ) lastdelta=$(( ${#revdeltas[@]} - cleanupno )) for (( i = ${#revdeltas[@]} - 1; i >= 0; i-- )); do - [[ ${i} != ${lastdelta} ]] || break + [[ ${i} != "${lastdelta}" ]] || break r=${revdeltas[${i}]} ldate=${r#*/${reponame}-} @@ -162,7 +163,96 @@ for algo in "${algo_LIST[@]}" ; do done # create checksums for snapshot and deltas -sha512sum -- *.sqfs *.sqdelta | \ - gpg --yes -u "${signkeyid}" --clearsign \ - --comment "Current: gentoo-${today}" --output sha512sum.txt.tmp - -mv sha512sum.txt.tmp sha512sum.txt +# OLD LOGIC, that scans entire 18GB; with 18GB of data this added 2 minutes of +# runtime onto a script that is otherwise under 30 seconds. +#date +ts-old-checksum-start=%s.%N +#ls -d -- *.sqfs *.sqdelta \ +#| xargs sha512sum -- \ +#| sort -k +2 \ +#| gpg \ +# --batch \ +# --yes \ +# -u "${signkeyid}" \ +# --clearsign \ +# --comment "Current: gentoo-${today}" \ +# --output sha512sum.txt.tmp \ +# /dev/stdin +#mv sha512sum.txt.tmp sha512sum.txt +#date +ts-old-checksum-end=%s.%N + +# NEW LOGIC, that tries to re-use signed checksums +# Helper func for signing. +sign_prefix() { + prefix=$1 + d=${prefix}.sha512sum.txt + + # Might be symlink OR real file. + find . \ + -name "${prefix}*" \ + -a \( -name "*.sqfs" -o -name '*.sqdelta' \) \ + -printf '%f\n' \ + | xargs sha512sum \ + | gpg --yes -u "${signkeyid}" --clearsign \ + --comment "Daily: ${prefix}" \ + --output "${d}.tmp" \ + /dev/stdin + + mv "${d}".tmp "${d}" +} + +date +ts-new-checksum-start=%s.%N +# 1. Create per-day checksums, with dates in the filenames; only if they do NOT +# exist. +find . -maxdepth 1 -mindepth 1 -name 'gentoo-*sqfs' -type f -printf '%f\n' \ +| cut -d. -f1 \ +| sort \ +| uniq \ +| perl -lne 'print $_ unless -e $_.".sha512sum.txt"' \ +| while read -r _prefix ; do + sign_prefix "$_prefix" +done +# 2. Always re-sign the -current symlink. +sign_prefix "gentoo-current" + +# 3. Re-verify each existing file +find . -maxdepth 1 -mindepth 1 -name 'gentoo-*sha512sum.txt' -printf '%f\n' \ +| while read -r sigfile ; do + rm -f "${tempdir}"/"${sigfile}".combine* + # FUTURE: it would be wonderful to have a better interface to GPG here that + # only sent the output if the signature was valid and matched the specified + # key. + # gpgv doesn't have assert-signer + if gpg --verify \ + --batch \ + --assert-signer "${signkeyid}" \ + --trusted-key "${signkeyid}" \ + --output "${tempdir}/${sigfile}".combine-tmp \ + --status-fd 3 \ + 2>"${tempdir}/${sigfile}".stderr \ + 3>"${tempdir}/${sigfile}".status-fd \ + "$sigfile" \ + ; then + mv "${tempdir}/${sigfile}".combine-tmp "${tempdir}/${sigfile}".combine-verified + else + echo "FATAL: $sigfile verification failed" 1>&2 + cat "${tempdir}/${sigfile}".stderr + cat "${tempdir}/${sigfile}".status-fd + fi +done + +# 4. Combine the verified files and sign the combined output. +find "$tempdir" -name 'gentoo-*combine-verified' \ + | xargs --no-run-if-empty cat \ + | sort -k +2 \ + | gpg \ + --batch \ + --yes \ + -u "${signkeyid}" \ + --clearsign \ + --comment "Current: gentoo-${today}" \ + --output "${tempdir}"/sha512sum.txt.tmp \ + /dev/stdin +# Move the final file into place +mv "${tempdir}"/sha512sum.txt.tmp sha512sum.txt +date +ts-new-checksum-done=%s.%N +date +ts-last=%s.%N diff --git a/rsync-gen.sh b/rsync-gen.sh index 82e72b9..8985377 100755 --- a/rsync-gen.sh +++ b/rsync-gen.sh @@ -209,28 +209,28 @@ date -u > ${STAGEDIR_repo_gentoo}/metadata/timestamp # 3) place dtd info in STAGEDIR_repo_gentoo timelog_start "DTD" | timelogger -rsync -Wqa --no-times --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/dtd ${STAGEDIR_repo_gentoo}/metadata/ +rsync -Wqa --no-times --checksum --exclude=.git --delete ${EXPORTS}/dtd ${STAGEDIR_repo_gentoo}/metadata/ date -R -u > ${STAGEDIR_repo_gentoo}/metadata/dtd/timestamp.chk timelog___end "DTD" | timelogger # end 3) # 3b) place xml schemas in STAGEDIR_repo_gentoo timelog_start "XML-SCHEMA" | timelogger -rsync -Wqa --no-times --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/xml-schema ${STAGEDIR_repo_gentoo}/metadata/ +rsync -Wqa --no-times --checksum --exclude=.git --delete ${EXPORTS}/xml-schema ${STAGEDIR_repo_gentoo}/metadata/ date -R -u > ${STAGEDIR_repo_gentoo}/metadata/xml-schema/timestamp.chk timelog___end "XML-SCHEMA" | timelogger # end 3b) # 4) place glsa's in STAGEDIR_repo_gentoo timelog_start "GLSA" | timelogger -rsync -Wqa --no-times --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/glsa ${STAGEDIR_repo_gentoo}/metadata/ +rsync -Wqa --no-times --checksum --exclude=.git --delete ${EXPORTS}/glsa ${STAGEDIR_repo_gentoo}/metadata/ date -R -u > ${STAGEDIR_repo_gentoo}/metadata/glsa/timestamp.chk timelog___end "GLSA" | timelogger # end 4) # 5) place news in STAGEDIR_repo_gentoo timelog_start "NEWS" | timelogger -rsync -Wqa --no-times --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/gentoo-news/. ${STAGEDIR_repo_gentoo}/metadata/news +rsync -Wqa --no-times --checksum --exclude=.git --delete ${EXPORTS}/gentoo-news/. ${STAGEDIR_repo_gentoo}/metadata/news date -R -u > ${STAGEDIR_repo_gentoo}/metadata/news/timestamp.chk timelog___end "NEWS" | timelogger # end 5) @@ -272,7 +272,7 @@ timelog_start "FINAL RSYNC" | timelogger FINALDIR_repo_gentoo_tmp=${FINALDIR_repo_gentoo%/} mkdir -p "${FINALDIR_repo_gentoo_tmp}-1" "${FINALDIR_repo_gentoo_tmp}-2" $atomic_rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \ - --exclude=.git --exclude=CVS --exclude=.gitignore \ + "--exclude=.*" \ --chmod=u-s,g-s --no-times \ ${STAGEDIR_repo_gentoo}/ \ ${FINALDIR_repo_gentoo}/ @@ -318,7 +318,7 @@ timelog___end "CACHE RSYNC" | timelogger # 9) rsync from STAGEDIR_repo_gentoo to REPODIR ### testing timelog_start "GIT RSYNC" | timelogger rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \ - --exclude=.git --exclude=CVS --exclude=.gitignore \ + "--exclude=.*" \ --chmod=u-s,g-s \ ${STAGEDIR_repo_gentoo}/ \ ${REPODIR}/ && \ diff --git a/sync-origin-mirror-releases.sh b/sync-origin-mirror-releases.sh index c0e6d4a..57ab8bf 100755 --- a/sync-origin-mirror-releases.sh +++ b/sync-origin-mirror-releases.sh @@ -1,11 +1,24 @@ #!/bin/bash # Copyright 2011-2015 Gentoo Authors; Distributed under the GPL v2 +h=$(hostname --fqdn |cut -d. -f1) + FINALDIR="/data/mirror" PASSWD_FILE="" RSYNC="/usr/bin/rsync" RSYNC_ARGS="--recursive --links --perms --times --delete --hard-links --no-motd --timeout=300 ${PASSWD_FILE:+--password-file }${PASSWD_FILE}" -RSYNC_ARGS="${RSYNC_ARGS} --quiet" +RSYNC_ARGS+=" --quiet" + +case $h in + kestrel) + # kestrel is space constrained, and the binpackages can be re-generated. + # Saves 100+ GB + RSYNC_ARGS+=" --exclude binpackages " + # kestrel is space constrained, and the snapshot squashfs are a nice-to-have: + # Saves 18 GB + RSYNC_ARGS+=" --exclude squashfs " + ;; +esac module=releases ${RSYNC} ${RSYNC_ARGS} masterreleases.gentoo.org::${module}/ ${FINALDIR}/${module}/ |