From cfa53785fada8ea1f270a28df858160ce88d295f Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 2 Jul 2026 20:08:48 +0000 Subject: [PATCH] fix(e2e): deliver image via k3d direct mode, keep retry + pin verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause found in k3d v5.9.0 source: executeInNode polls ContainerExecInspect and trusts Running=false/ExitCode=0, but Docker's exec lifecycle is async — the poll can misread the tools-node save-image exec as complete before the tar exists on the shared volume, and importWithToolsNode only logs (never returns) the resulting per-node ctr import failure, so k3d exits 0 claiming success. Direct mode streams docker save into each node's ctr stdin: no tarball, no shared-volume window, and node import failures are propagated. Verified on a live 4-node cluster with the project image. Caveat documented in-script: direct + ctr --all-platforms cannot handle multi-arch images saved by a containerd-store docker; PROJECT_IMAGE is always single-arch. The retry now also covers a loud nonzero exit from k3d, and the pin step remains the final verification that the image really landed. Co-Authored-By: Claude Fable 5 --- hack/e2e/load-image.sh | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/hack/e2e/load-image.sh b/hack/e2e/load-image.sh index dc60f83b..f675df58 100755 --- a/hack/e2e/load-image.sh +++ b/hack/e2e/load-image.sh @@ -165,21 +165,33 @@ runtime_image_id_in_node() { import_image() { local ref="$1" local attempt node_name pinned - # k3d's tools-node import path can transiently lose the tarball between the - # save and the per-node `ctr image import` ("ctr: open /k3d/images/: - # no such file or directory") — and k3d still exits 0, logging "Successfully - # imported". The pin step below is what actually detects the miss, so on a - # failed pin re-run the whole import instead of giving up. + # Deliver with `-m direct` (docker save streamed straight into each node's + # `ctr image import` stdin) rather than the default tools-node mode. The + # tools-node path has a race: k3d's exec poll can misread the save-image + # exec as finished before the tarball hits the shared volume, the node + # import then fails ("ctr: open /k3d/images/: no such file or + # directory") — and k3d logs the error but still exits 0. Direct mode has + # no tarball/volume window and DOES propagate node import failures. + # Caveat: direct + ctr's --all-platforms chokes on multi-arch images saved + # by a containerd-store docker (foreign-platform blobs absent from the + # stream); PROJECT_IMAGE is always the single-arch controller image. + # + # Trust nothing either way: the pin step below verifies the image really + # landed in every node, and any failed attempt (loud or silent) is retried. for attempt in 1 2 3; do echo "Importing ${PROJECT_IMAGE} into k3d cluster ${CLUSTER_NAME} (attempt ${attempt}/3)" - "${K3D}" image import -c "${CLUSTER_NAME}" "${PROJECT_IMAGE}" pinned=1 - while IFS= read -r node_name; do - if ! pin_imported_image "${node_name}" "${ref}" "${PROJECT_IMAGE}" "${IMAGE_REPO}" "${IMAGE_TAG}"; then - pinned=0 - break - fi - done < <(cluster_node_names) + if "${K3D}" image import -m direct -c "${CLUSTER_NAME}" "${PROJECT_IMAGE}"; then + while IFS= read -r node_name; do + if ! pin_imported_image "${node_name}" "${ref}" "${PROJECT_IMAGE}" "${IMAGE_REPO}" "${IMAGE_TAG}"; then + pinned=0 + break + fi + done < <(cluster_node_names) + else + echo "WARN: k3d image import exited nonzero on attempt ${attempt}" >&2 + pinned=0 + fi if [[ "${pinned}" == "1" ]]; then return 0 fi