From f0e02868e2e203fbb29ee08eecda734366575079 Mon Sep 17 00:00:00 2001 From: James Fredley Date: Fri, 3 Jul 2026 22:56:34 -0400 Subject: [PATCH 1/2] fix: support concurrent documentation deploys to GitHub Pages When multiple release/snapshot builds publish documentation at the same time, their pushes to the documentation branch race. Previously a rejected push triggered a plain `git pull --rebase`, which stops on a merge conflict in the shared folders (latest, X.X.x, index.html, snapshot) and, under `set -e`, kills the whole job - so concurrent releases fail. Make the retry loop concurrency-safe: - Rebase the deploy commit onto the remote with `-X theirs` so shared-folder file conflicts auto-resolve in favour of this deploy, and add bounded backoff with jitter between attempts. - Reconcile owned paths after the rebase: purge-owned folders are restored wholesale from the deploy commit (clean last-writer-wins) while merge-mode folders and other deploys' independent version folders are preserved. - Keep `latest` as highest-version-wins: a lower release replayed after a concurrent higher release yields `latest` back to the remote instead of overwriting it with older docs. - Resolve modify/delete rebase conflicts that `-X theirs` cannot handle (a file this deploy purges but a concurrent deploy modified) by driving the rebase to completion deterministically instead of aborting. Add a Testcontainers regression test that lands a concurrent deploy modifying a purged file and asserts the retry recovers cleanly. Assisted-by: claude-code:claude-4.8-opus --- deploy-github-pages/entrypoint.sh | 189 +++++++++++++++++- .../github/DeployGithubPagesSpec.groovy | 85 +++++++- 2 files changed, 270 insertions(+), 4 deletions(-) diff --git a/deploy-github-pages/entrypoint.sh b/deploy-github-pages/entrypoint.sh index 01eaa2b..7075198 100755 --- a/deploy-github-pages/entrypoint.sh +++ b/deploy-github-pages/entrypoint.sh @@ -91,7 +91,16 @@ publish_artifacts() { echo "Removing existing file at ${purgePath}" git rm -rf "${purgePath}" fi + + # This deploy replaces ${purgePath} wholesale, so on a concurrent-deploy retry + # it must be restored to exactly this content (clean last-writer-wins). + OWNED_PURGE_PATHS+=("${purgePath}") fi + # Note: when PURGE_EXISTING=false this deploy MERGES into ${PUBLISH_PATH}, so it + # is deliberately NOT reconciled - the "-X theirs" rebase already keeps a + # concurrent deploy's independent files while winning any file-level conflict, + # which is exactly the intended merge semantics. Re-asserting the whole folder + # here would wrongly revert the other deploy's changes to files we also touched. echo "Publishing ${SOURCE_FOLDER} to ${DOCUMENTATION_BRANCH}:${PUBLISH_PATH}" mkdir -p "${PUBLISH_PATH}" @@ -130,6 +139,140 @@ is_highest_version() { return 0 } +# Paths this deploy owns. Populated by publish_artifacts and used to reconcile the +# working tree after rebasing onto a concurrent deploy (see reconcile_owned_paths). +# OWNED_PURGE_PATHS - folders this deploy replaces wholesale (PURGE_EXISTING=true): +# on retry, strip anything a concurrent deploy left there and +# restore exactly this deploy's content (clean last-writer-wins). +# OWNED_KEEP_PATHS - single files this deploy owns outright (e.g. the root +# index.html): on retry, re-assert just that file. Merged +# folders (PURGE_EXISTING=false) are intentionally NOT listed +# here - the "-X theirs" rebase already gives correct merge +# semantics, and re-asserting them would clobber a concurrent +# deploy's independent changes. +OWNED_PURGE_PATHS=() +OWNED_KEEP_PATHS=() + +# If this release publishes the shared "latest" folder, remember its path and this +# release's generic (X.X.x) version. "latest" must always hold the HIGHEST released +# version, but is_highest_version is evaluated against the checkout taken before a +# concurrent higher release may have landed. reconcile_owned_paths re-checks these +# against the rebased tree so a lower release retried after a higher one does not +# overwrite "latest" with older docs. Empty unless this deploy published "latest". +LATEST_OWNED_PATH="" +LATEST_GENERIC_VERSION="" + +# After a push is rejected we rebase this deploy's commit onto the concurrent deploy +# with "-X theirs", which wins FILE-LEVEL conflicts but cannot remove files the other +# deploy ADDED to a folder we own. Re-assert our owned paths from our original commit +# so shared folders (latest, X.X.x, ...) end up as a clean last-writer-wins instead of +# a union of both deploys' files. Everything the other deploy published independently +# (its own version folders) is untouched and preserved. +reconcile_owned_paths() { + local p changed=false + + # "latest" must hold the HIGHEST released version. If a concurrent, higher + # release landed first (its X.X.x folder is now in the just-rebased tree) and + # THIS is a lower release, do not overwrite "latest" with our older docs: + # restore it to the remote's (higher) content and drop it from our owned set. + # Re-run is_highest_version from within TARGET_FOLDER, where the version folders + # actually live, against the merged tree. Same-minor patches compare equal, so + # they still fall through to last-writer-wins, which is fine. + if [ -n "${LATEST_OWNED_PATH}" ] && [ -n "${LATEST_GENERIC_VERSION}" ]; then + if ! ( cd "${TARGET_FOLDER}" && is_highest_version "${LATEST_GENERIC_VERSION}" ); then + echo "A concurrent higher release now owns ${LATEST_OWNED_PATH}; yielding it to the remote." + # Strip our rebased copy first so files that exist ONLY in our (lower) + # version are removed - otherwise the checkout below would leave them and + # "latest" would become a union of both releases instead of exactly the + # remote's higher version. FETCH_HEAD is the remote tip fetched by the + # preceding `git pull --rebase`, i.e. the concurrent deploy we rebased onto. + git rm -rf --ignore-unmatch --quiet -- "${LATEST_OWNED_PATH}" >/dev/null 2>&1 || true + git checkout FETCH_HEAD -- "${LATEST_OWNED_PATH}" >/dev/null 2>&1 || true + git add -A -- "${LATEST_OWNED_PATH}" >/dev/null 2>&1 || true + local kept=() x + for x in "${OWNED_PURGE_PATHS[@]}"; do + [ "$x" = "${LATEST_OWNED_PATH}" ] || kept+=("$x") + done + OWNED_PURGE_PATHS=("${kept[@]}") + changed=true + fi + fi + + for p in "${OWNED_PURGE_PATHS[@]}"; do + git rm -rf --ignore-unmatch --quiet -- "$p" >/dev/null 2>&1 || true + git checkout "$DEPLOY_COMMIT" -- "$p" >/dev/null 2>&1 || true + git add -A -- "$p" >/dev/null 2>&1 || true + changed=true + done + for p in "${OWNED_KEEP_PATHS[@]}"; do + git checkout "$DEPLOY_COMMIT" -- "$p" >/dev/null 2>&1 || true + git add -A -- "$p" >/dev/null 2>&1 || true + changed=true + done + if [ "$changed" = true ]; then + echo "Re-asserted this deploy's owned paths after rebase." + # A fresh commit (not --amend) so this is safe even when resolve_rebase_conflicts + # had to `git rebase --skip` an emptied deploy commit, leaving HEAD at the remote + # tip - amending there would rewrite the concurrent deploy's commit. + git commit --quiet --allow-empty -m "Reconcile documentation paths after concurrent deploy" + fi +} + +# `git pull --rebase -X theirs` auto-resolves file-CONTENT conflicts in favour of this +# deploy, but it canNOT auto-resolve modify/delete conflicts - e.g. this deploy purged a +# file from a shared folder (latest, X.X.x, snapshot) that a concurrent deploy modified. +# Such a conflict halts the rebase mid-way; without handling it the retry loop just aborts +# and re-hits the identical conflict until it exhausts all attempts. Finish the rebase +# deterministically instead: for every still-conflicted path take THIS deploy's version +# from DEPLOY_COMMIT, or drop it if this deploy does not have it, then continue. +# reconcile_owned_paths afterwards still restores owned folders wholesale. Returns 0 if a +# rebase was in progress and was driven to completion, 1 otherwise (e.g. the pull failed +# for a non-conflict reason such as network/auth, so the caller must not proceed). +resolve_rebase_conflicts() { + local git_dir + git_dir="$(git rev-parse --git-dir 2>/dev/null)" || return 1 + if [ ! -d "${git_dir}/rebase-merge" ] && [ ! -d "${git_dir}/rebase-apply" ]; then + return 1 + fi + + echo "Rebase halted on a conflict '-X theirs' cannot auto-resolve (e.g. a file this" >&2 + echo "deploy removed but a concurrent deploy modified). Resolving each conflicted path" >&2 + echo "in favour of THIS deploy and continuing the rebase." >&2 + + local guard=0 unmerged f + while [ -d "${git_dir}/rebase-merge" ] || [ -d "${git_dir}/rebase-apply" ]; do + guard=$((guard + 1)) + if [ "${guard}" -gt 100 ]; then + echo "Gave up resolving rebase conflicts after ${guard} iterations." >&2 + git rebase --abort >/dev/null 2>&1 || true + return 1 + fi + + unmerged="$(git diff --name-only --diff-filter=U 2>/dev/null)" + if [ -n "${unmerged}" ]; then + while IFS= read -r f; do + [ -n "${f}" ] || continue + if git cat-file -e "${DEPLOY_COMMIT}:${f}" 2>/dev/null; then + git checkout "${DEPLOY_COMMIT}" -- "${f}" >/dev/null 2>&1 || true + git add -- "${f}" >/dev/null 2>&1 || true + else + git rm -f --ignore-unmatch --quiet -- "${f}" >/dev/null 2>&1 || true + fi + done <<< "${unmerged}" + fi + + if ! GIT_EDITOR=: git rebase --continue >/dev/null 2>&1; then + # The replayed commit may now be empty (all its changes already exist upstream). + # Drop it; reconcile_owned_paths re-asserts this deploy's owned paths afterwards. + if ! GIT_EDITOR=: git rebase --skip >/dev/null 2>&1; then + git rebase --abort >/dev/null 2>&1 || true + return 1 + fi + fi + done + return 0 +} + set -e set_value_or_error "${DOCUMENTATION_BRANCH}" 'gh-pages' 'DOCUMENTATION_BRANCH' @@ -225,6 +368,7 @@ if [[ -f "../${SOURCE_FOLDER}/ghpages.html" ]]; then echo "${SOURCE_FOLDER}/ghpages.html detected, replacing root index.html" cp "../${SOURCE_FOLDER}/ghpages.html" index.html git add index.html + OWNED_KEEP_PATHS+=("index.html") echo "::endgroup::" fi @@ -297,6 +441,16 @@ else fi publish_artifacts echo "Published a copy of documentation to ${PUBLISH_PATH}" + # Record the "latest" folder so reconcile_owned_paths can re-check, after a + # concurrent-deploy rebase, whether this release is still the highest. Even + # merge-mode deployments need this check: a lower release replayed after a + # concurrent higher release must yield "latest" back to the higher release. + LATEST_GENERIC_VERSION="${genericVersionFolder}" + if [[ "${PURGE_EXISTING}" == "true" && "${PURGE_BY_BASE_PATH}" == "true" ]]; then + LATEST_OWNED_PATH="${BASE_PUBLISH_PATH}" + else + LATEST_OWNED_PATH="${PUBLISH_PATH}" + fi echo "::endgroup::" else echo "::group::Skipped Latest Release Documentation - not highest" @@ -317,11 +471,15 @@ git status echo "Committing changes." git commit -m "Deploying to documentation branch - $(date +"%T")" --quiet --allow-empty +# Remember this deploy's own commit. After a rebase rewrites it on top of a +# concurrent deploy, reconcile_owned_paths restores our owned folders from here. +DEPLOY_COMMIT="$(git rev-parse HEAD)" + MAX_PUSH_ATTEMPTS=5 PUSH_ATTEMPT=1 while [ $PUSH_ATTEMPT -le $MAX_PUSH_ATTEMPTS ]; do echo "Push attempt ${PUSH_ATTEMPT}/${MAX_PUSH_ATTEMPTS}" - if git push "${GIT_REPO_URL}" "${DOCUMENTATION_BRANCH}" 2>&1; then + if git push "${GIT_REPO_URL}" "${DOCUMENTATION_BRANCH}"; then echo "Deployment successful!" break fi @@ -329,8 +487,33 @@ while [ $PUSH_ATTEMPT -le $MAX_PUSH_ATTEMPTS ]; do echo "ERROR: Push failed after ${MAX_PUSH_ATTEMPTS} attempts." >&2 exit 1 fi - echo "Push rejected, pulling remote changes and retrying..." - git pull --rebase "${GIT_REPO_URL}" "${DOCUMENTATION_BRANCH}" + + # The push was rejected because a concurrent deploy (e.g. another release + # publishing at the same time) advanced the documentation branch. Replay this + # deploy's commit on top of the remote so both survive. + # + # Specific-version folders (the exact X.X.X) never collide across different + # releases; the paths that DO overlap are the shared ones - "latest", the + # generic "X.X.x" (every patch of a minor writes it), index.html, and the + # snapshot folder. Rebase with "-X theirs" to auto-resolve those file-level + # conflicts in favour of THIS deploy (during a rebase, "theirs" is the commit + # being replayed). Without this the rebase stops on a merge conflict and, + # under `set -e`, kills the whole job - which is why concurrent releases fail + # today. "-X theirs" wins conflicting files but cannot remove files the other + # deploy ADDED to a folder we own, so reconcile_owned_paths (below) then + # re-asserts our owned folders to a clean last-writer-wins. + echo "Push rejected, rebasing onto remote changes and retrying..." PUSH_ATTEMPT=$((PUSH_ATTEMPT + 1)) + BACKOFF=$(( (PUSH_ATTEMPT * 2) + (RANDOM % 3) )) + echo "Waiting ${BACKOFF}s before retry..." + sleep $BACKOFF + if git pull --rebase -X theirs "${GIT_REPO_URL}" "${DOCUMENTATION_BRANCH}"; then + reconcile_owned_paths + elif resolve_rebase_conflicts; then + reconcile_owned_paths + else + echo "Rebase could not be completed automatically; aborting before the next attempt." >&2 + git rebase --abort >/dev/null 2>&1 || true + fi done echo "::endgroup::" diff --git a/tests/src/test/groovy/org/apache/grails/github/DeployGithubPagesSpec.groovy b/tests/src/test/groovy/org/apache/grails/github/DeployGithubPagesSpec.groovy index 1040c5b..63772ed 100644 --- a/tests/src/test/groovy/org/apache/grails/github/DeployGithubPagesSpec.groovy +++ b/tests/src/test/groovy/org/apache/grails/github/DeployGithubPagesSpec.groovy @@ -720,7 +720,7 @@ exec "$REAL_GIT" "$@" and: 'first push was rejected and retry occurred' action.actionLogs.contains('Push attempt 1/5') - action.actionLogs.contains('Push rejected, pulling remote changes and retrying...') + action.actionLogs.contains('Push rejected, rebasing onto remote changes and retrying...') action.actionLogs.contains('Push attempt 2/5') action.actionLogs.contains('Deployment successful!') @@ -733,6 +733,89 @@ exec "$REAL_GIT" "$@" System.out.println("Container logs:\n${action.actionLogs}" as String) } + def "push retry - resolves modify/delete conflict from a concurrent deploy"() { + given: + GitHubVersion release = new GitHubVersion(version: '7.0.0-RC1', tagName: 'rel-7.0.0-RC1', targetBranch: '7.0.x', targetVersion: '7.0.0-SNAPSHOT') + action = new GitHubDockerAction('deploy-github-pages', release, new GitHubCliMock()) + + gitRepo = new GitHubRepoMock(action.workspacePath, net) + gitRepo.init() + gitRepo.populateRepository('7.0.0-SNAPSHOT', null, [], getProjectFiles()) + gitRepo.createDivergedBranch([ + 'index.html' : 'Existing root page', + 'snapshot/index.html' : 'Existing snapshot', + 'snapshot/legacy.html': 'Legacy snapshot page' + ], 'gh-pages') + gitRepo.stageRepositoryForAction('main', false) + + and: 'a git wrapper that, on the first push, lands a concurrent deploy MODIFYING a file this deploy PURGES (a modify/delete conflict), then rejects our push' + def gitWrapper = action.mockPath.resolve('git').toFile() + gitWrapper.text = '''\ +#!/bin/sh +REAL_GIT=/usr/bin/git +MARKER=/tmp/git_push_rejected_once +if [ "$1" = "push" ]; then + if [ ! -f "$MARKER" ]; then + touch "$MARKER" + REMOTE="$2" + BRANCH="$3" + TMP=$(mktemp -d) + if "$REAL_GIT" clone --branch "$BRANCH" --single-branch "$REMOTE" "$TMP/r" >/dev/null 2>&1; then + echo 'Concurrently modified legacy page' > "$TMP/r/snapshot/legacy.html" + cd "$TMP/r" + "$REAL_GIT" config user.email concurrent@example.com + "$REAL_GIT" config user.name concurrent + "$REAL_GIT" add -A >/dev/null 2>&1 + "$REAL_GIT" commit -m "concurrent deploy" >/dev/null 2>&1 + "$REAL_GIT" push "$REMOTE" "$BRANCH" >/dev/null 2>&1 + cd / + fi + rm -rf "$TMP" + echo "error: failed to push some refs" >&2 + echo "hint: Updates were rejected because the remote contains work that you do not have locally." >&2 + exit 1 + fi +fi +exec "$REAL_GIT" "$@" +''' + gitWrapper.executable = true + + and: + def env = getDefaultEnvironment(action, gitRepo) + env['GRADLE_PUBLISH_RELEASE'] = 'false' + env['SOURCE_FOLDER'] = 'docs' + env['VERSION'] = '7.0.0-SNAPSHOT' + + and: + action.createContainer(env, net) + + when: + action.runAction() + + then: 'the action recovers and deploys successfully despite the modify/delete conflict' + action.actionExitCode == 0L + + and: 'the retry rebased onto the concurrent deploy' + action.actionLogs.contains('Push attempt 1/5') + action.actionLogs.contains('Push rejected, rebasing onto remote changes and retrying...') + action.actionLogs.contains('Push attempt 2/5') + action.actionLogs.contains('Deployment successful!') + + and: 'this deploy owns the snapshot folder wholesale - its new content is present' + gitRepo.branchExists('gh-pages') + gitRepo.getFileContents('snapshot/index.html', 'gh-pages') == 'Welcome to the Grails Documentation' + gitRepo.getFileContents('index.html', 'gh-pages') == 'Welcome to the Grails GitHub Pages' + + when: 'the purged legacy file is looked up' + gitRepo.getFileContents('snapshot/legacy.html', 'gh-pages') + + then: 'it was removed despite the concurrent modification (clean last-writer-wins, no modify/delete stall)' + thrown(IllegalStateException) + + cleanup: + System.out.println("Container logs:\n${action.actionLogs}" as String) + } + def "push retry - fails after maximum push attempts"() { given: GitHubVersion release = new GitHubVersion(version: '7.0.0-RC1', tagName: 'rel-7.0.0-RC1', targetBranch: '7.0.x', targetVersion: '7.0.0-SNAPSHOT') From b1708513c90d9f481aa3171e3b04a1fc6a994c62 Mon Sep 17 00:00:00 2001 From: a Date: Fri, 3 Jul 2026 23:30:27 -0400 Subject: [PATCH 2/2] fix: fail the deploy instead of masking reconcile restore errors Addresses review feedback on the concurrent-deploy reconcile logic. The authoritative restore commands in reconcile_owned_paths previously used `|| true`, so if restoring an owned documentation path failed, the action would silently stage and commit a deletion and publish broken/missing docs. - Owned purge/keep paths always exist in the deploy commit, so their `git checkout`/`git add` restores are now unmasked: a failure fails the job under `set -e` instead of committing a silent deletion. Only the best-effort pre-clean `git rm --ignore-unmatch` stays lenient. - The "latest" yield path only restores from the remote when the remote actually publishes it (guarded by `git ls-tree`); otherwise the staged removal stands, avoiding an abort on a now-absent pathspec. The `git ls-tree` check is unmasked so an invalid FETCH_HEAD fails closed. Assisted-by: claude-code:claude-4.8-opus --- deploy-github-pages/entrypoint.sh | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/deploy-github-pages/entrypoint.sh b/deploy-github-pages/entrypoint.sh index 7075198..fe24214 100755 --- a/deploy-github-pages/entrypoint.sh +++ b/deploy-github-pages/entrypoint.sh @@ -187,8 +187,19 @@ reconcile_owned_paths() { # remote's higher version. FETCH_HEAD is the remote tip fetched by the # preceding `git pull --rebase`, i.e. the concurrent deploy we rebased onto. git rm -rf --ignore-unmatch --quiet -- "${LATEST_OWNED_PATH}" >/dev/null 2>&1 || true - git checkout FETCH_HEAD -- "${LATEST_OWNED_PATH}" >/dev/null 2>&1 || true - git add -A -- "${LATEST_OWNED_PATH}" >/dev/null 2>&1 || true + # Restore the remote's (higher) version IF the remote actually publishes this + # path. If it does not (e.g. the higher release skipped "latest"), the + # `git rm` above already staged its removal - there is nothing to restore, and + # a `git add -A` on the now-absent pathspec would exit 128 and abort. A + # genuinely broken FETCH_HEAD makes `git ls-tree` exit nonzero, which - left + # unmasked - fails the job under `set -e` rather than being mistaken for + # "path absent". A checkout failure on a path the remote DOES have also aborts. + local remote_latest + remote_latest="$(git ls-tree -r FETCH_HEAD -- "${LATEST_OWNED_PATH}")" + if [ -n "${remote_latest}" ]; then + git checkout FETCH_HEAD -- "${LATEST_OWNED_PATH}" + git add -A -- "${LATEST_OWNED_PATH}" + fi local kept=() x for x in "${OWNED_PURGE_PATHS[@]}"; do [ "$x" = "${LATEST_OWNED_PATH}" ] || kept+=("$x") @@ -198,15 +209,20 @@ reconcile_owned_paths() { fi fi + # These owned paths are always present in DEPLOY_COMMIT (this deploy published + # them), so a restore failure means a genuinely broken repo state. Do NOT mask + # checkout/add failures with `|| true`: under `set -e` they must fail the job + # rather than silently commit a deletion and publish broken/missing docs. Only + # the best-effort pre-clean `git rm` (guarded by --ignore-unmatch) stays lenient. for p in "${OWNED_PURGE_PATHS[@]}"; do git rm -rf --ignore-unmatch --quiet -- "$p" >/dev/null 2>&1 || true - git checkout "$DEPLOY_COMMIT" -- "$p" >/dev/null 2>&1 || true - git add -A -- "$p" >/dev/null 2>&1 || true + git checkout "$DEPLOY_COMMIT" -- "$p" + git add -A -- "$p" changed=true done for p in "${OWNED_KEEP_PATHS[@]}"; do - git checkout "$DEPLOY_COMMIT" -- "$p" >/dev/null 2>&1 || true - git add -A -- "$p" >/dev/null 2>&1 || true + git checkout "$DEPLOY_COMMIT" -- "$p" + git add -A -- "$p" changed=true done if [ "$changed" = true ]; then