Speed up lang-SDK k8s test by building Go/Java natively in CI

.github/workflows/ci-amd.yml

@@ -99,6 +99,7 @@ jobs:
       include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
       individual-providers-test-types-list-as-strings-in-json: >-
         ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }}
+      java-sdk-version: ${{ steps.selective-checks.outputs.java-sdk-version }}
       kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }}
       kubernetes-combos-list-as-string: >-
         ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
@@ -932,6 +933,7 @@ jobs:
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
       kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }}
+      java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
     if: >
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
       needs.build-info.outputs.run-helm-tests == 'true')

.github/workflows/ci-arm.yml

@@ -92,6 +92,7 @@ jobs:
       include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
       individual-providers-test-types-list-as-strings-in-json: >-
         ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }}
+      java-sdk-version: ${{ steps.selective-checks.outputs.java-sdk-version }}
       kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }}
       kubernetes-combos-list-as-string: >-
         ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
@@ -925,6 +926,7 @@ jobs:
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
       kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }}
+      java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
     if: >
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
       needs.build-info.outputs.run-helm-tests == 'true')

.github/workflows/k8s-tests.yml

@@ -48,6 +48,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources"
         required: true
         type: string
+      java-sdk-version:
+        description: "JDK version used to build the lang-SDK Java artifacts natively in CI"
+        required: true
+        type: string
 permissions:
   contents: read
 jobs:
@@ -96,6 +100,45 @@ jobs:
           make-mnt-writeable-and-cleanup: true
         id: breeze
       # preparing k8s environment with uv takes < 15 seconds with `uv` - there is no point in caching it.
+      # Provision the lang-SDK Go/Java toolchains on the host only for the single variant that runs the
+      # lang-SDK test (KubernetesExecutor, standard-naming off). LANG_SDK_NATIVE_TOOLCHAIN=true then makes
+      # breeze build the artifacts natively instead of in throwaway toolchain containers, skipping the
+      # image pulls and reusing the caches restored below. The module/build and Gradle caches are keyed
+      # explicitly (rather than via the setup-* built-in caching) so the key carries an ``-vN-`` salt:
+      # bump it to force-invalidate a poisoned cache without waiting for a dependency change to rotate it.
+      # keep go-version in sync with go-sdk/go.mod and the sibling Go SDK job in ci-amd.yml.
+      - name: "Setup Go for lang-SDK build"
+        if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16  # v6.5.0
+        with:
+          go-version: 1.25
+          cache: false
+      - name: "Cache lang-SDK Go module + build cache"
+        if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: lang-sdk-go-v1-${{ runner.os }}-${{ hashFiles('kubernetes-tests/lang_sdk/go_example/go.sum') }}
+          restore-keys: |
+            lang-sdk-go-v1-${{ runner.os }}-
+      - name: "Setup Java for lang-SDK build"
+        if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false
+        uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520  # v5.4.0
+        with:
+          distribution: 'temurin'
+          java-version: ${{ inputs.java-sdk-version }}
+      - name: "Cache lang-SDK Gradle distribution + dependencies"
+        if: matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: lang-sdk-gradle-v1-${{ runner.os }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle-wrapper.properties', 'kubernetes-tests/lang_sdk/java_example/**/*.gradle*') }}  # yamllint disable-line rule:line-length
+          restore-keys: |
+            lang-sdk-gradle-v1-${{ runner.os }}-
       - name: "\
           Run complete K8S tests ${{ matrix.executor }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-\
           ${{env.KUBERNETES_VERSION}}-${{ matrix.use-standard-naming }}"
@@ -106,6 +149,7 @@ jobs:
           # Provision + run the lang-SDK coordinator system test in a single variant only
           # (KubernetesExecutor, standard-naming off) to keep it off the other five k8s jobs.
           RUN_LANG_SDK_K8S_TESTS: ${{ (matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false) && 'true' || 'false' }}  # yamllint disable-line rule:line-length
+          LANG_SDK_NATIVE_TOOLCHAIN: ${{ (matrix.executor == 'KubernetesExecutor' && matrix.use-standard-naming == false) && 'true' || 'false' }}  # yamllint disable-line rule:line-length
           VERBOSE: "false"
       - name: "\
           Print logs ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\

dev/breeze/doc/ci/04_selective_checks.md

@@ -531,6 +531,7 @@ GitHub Actions to pass the list of parameters to a command to execute
 | individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed)                       | Providers[\amazon\] Providers\[google\]  | *    |
 | is-committer-build                                      | Whether the build is triggered by a committer                                                           | false                                    |      |
 | is-legacy-ui-api-labeled                                | Whether the PR is labeled as legacy UI/API                                                              | false                                    |      |
+| java-sdk-version                                        | JDK version used to build the lang-SDK Java artifacts natively in CI                                     | 17                                       |      |
 | kind-version                                            | Which Kind version to use for tests                                                                     | v0.24.0                                  |      |
 | kubernetes-combos-list-as-string                        | All combinations of Python version and Kubernetes version to use for tests as space-separated string    | 3.10-v1.25.2 3.11-v1.28.13               | *    |
 | kubernetes-versions                                     | All Kubernetes versions to use for tests as JSON array                                                  | \['v1.25.2'\]                            |      |

dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py

@@ -2509,123 +2509,153 @@ def deploy_cluster(
 )
 
 
-def _lang_sdk_build_go_bundle(staging: Path, output: Output | None) -> None:
-    """Build the Go bundle into ``staging/go-artifacts`` in an ephemeral Go toolchain container.
+def _lang_sdk_build_go_bundle(staging: Path, output: Output | None, *, native: bool = False) -> None:
+    """Build the Go bundle into ``staging/go-artifacts`` and copy the result into the staging dir.
 
-    The container means the host needs no Go install; the build writes into a gitignored dir under
-    the repo (with persistent caches) and the result is copied into the staging dir.
+    By default the build runs in an ephemeral Go toolchain container so the host needs no Go install,
+    writing its caches into a gitignored dir under the repo. In ``native`` mode (used in CI, where the
+    host already has a cached Go toolchain via ``actions/setup-go``) it invokes the host ``go`` directly,
+    skipping the container image pull and reusing the runner's module/build cache.
     """
     go_dir = staging / "go-artifacts"
     go_dir.mkdir(parents=True, exist_ok=True)
+    output_bin = LANG_SDK_GO_EXAMPLE_PATH / "bin" / LANG_SDK_GO_BUNDLE_NAME
 
-    uid_gid = f"{os.getuid()}:{os.getgid()}"
-    go_example_ctr = f"/repo/{LANG_SDK_GO_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}"
-
-    # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker.
-    # USER/HOME must be set because the SDK calls user.Current() at init; with
-    # cgo disabled Go's pure-Go resolver reads those env vars and panics if
-    # either is empty. HOME points at a writable, gitignored dir under go_example
-    # so the Go build and module caches persist between runs. The package built is
+    # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. The package built is
     # the current dir (".") because go_example is its own module.
-    get_console(output=output).print(f"[info]Building Go bundle in {LANG_SDK_GO_BUILDER_IMAGE}")
-    run_command(
-        [
+    if native:
+        get_console(output=output).print("[info]Building Go bundle with the host Go toolchain")
+        run_command(
+            ["go", "tool", "airflow-go-pack", "--output", str(output_bin), "."],
+            cwd=LANG_SDK_GO_EXAMPLE_PATH,
+            env={**os.environ, "CGO_ENABLED": "0"},
+            output=output,
+            check=True,
+        )
+    else:
+        uid_gid = f"{os.getuid()}:{os.getgid()}"
+        go_example_ctr = f"/repo/{LANG_SDK_GO_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}"
+        # USER/HOME must be set because the SDK calls user.Current() at init; with cgo disabled Go's
+        # pure-Go resolver reads those env vars and panics if either is empty. HOME points at a
+        # writable, gitignored dir under go_example so the caches persist between runs on a dev host.
+        get_console(output=output).print(f"[info]Building Go bundle in {LANG_SDK_GO_BUILDER_IMAGE}")
+        run_command(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--user",
+                uid_gid,
+                "-e",
+                f"HOME={go_example_ctr}/.home",
+                "-e",
+                "USER=airflow",
+                "-e",
+                "CGO_ENABLED=0",
+                "-v",
+                f"{AIRFLOW_ROOT_PATH}:/repo",
+                "-w",
+                go_example_ctr,
+                LANG_SDK_GO_BUILDER_IMAGE,
+                "go",
+                "tool",
+                "airflow-go-pack",
+                "--output",
+                f"{go_example_ctr}/bin/{LANG_SDK_GO_BUNDLE_NAME}",
+                ".",
+            ],
+            output=output,
+            check=True,
+        )
+    shutil.copy(output_bin, go_dir / LANG_SDK_GO_BUNDLE_NAME)
+
+
+def _lang_sdk_build_java_jar(staging: Path, output: Output | None, *, native: bool = False) -> None:
+    """Publish the Java SDK to mavenLocal then build the java_example jar into ``staging/java-artifacts``.
+
+    By default the build runs in an ephemeral JDK container so the host needs no JDK, persisting the
+    Gradle distribution/dependency and Maven caches via mounted dirs. In ``native`` mode (used in CI,
+    where the host already has a cached JDK + Gradle cache via ``actions/setup-java``) it invokes the
+    host ``./gradlew`` directly, skipping the container image pull and reusing the runner's ``~/.gradle``
+    cache. ``java_example`` resolves the SDK from ``mavenLocal()``, so the SDK is published first, then
+    the bundle is built with java-sdk's gradle wrapper pointed at the example project (``-p``).
+    """
+    java_dir = staging / "java-artifacts"
+    java_dir.mkdir(parents=True, exist_ok=True)
+    java_sdk_path = AIRFLOW_ROOT_PATH / "java-sdk"
+
+    if native:
+        get_console(output=output).print(
+            "[info]Publishing Java SDK artifacts to local Maven repository with the host Gradle toolchain"
+        )
+        run_command(
+            ["./gradlew", "publishToMavenLocal", "-PskipSigning=true", "--no-daemon", "--console=plain"],
+            cwd=java_sdk_path,
+            output=output,
+            check=True,
+        )
+        get_console(output=output).print("[info]Building Java jar with the host Gradle toolchain")
+        run_command(
+            ["./gradlew", "-p", str(LANG_SDK_JAVA_EXAMPLE_PATH), "bundle", "--no-daemon", "--console=plain"],
+            cwd=java_sdk_path,
+            output=output,
+            check=True,
+        )
+    else:
+        uid_gid = f"{os.getuid()}:{os.getgid()}"
+        java_example_ctr = f"/repo/{LANG_SDK_JAVA_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}"
+        # --user keeps build outputs owned by the host user; HOME is set explicitly because that UID has
+        # no /etc/passwd entry; GRADLE_USER_HOME and the mounted ~/.m2 persist the Gradle distribution and
+        # dependency caches between runs on a dev host.
+        LANG_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+        java_docker_prefix = [
             "docker",
             "run",
             "--rm",
             "--user",
             uid_gid,
             "-e",
-            f"HOME={go_example_ctr}/.home",
+            "GRADLE_USER_HOME=/repo/java-sdk/.gradle",
             "-e",
-            "USER=airflow",
-            "-e",
-            "CGO_ENABLED=0",
+            "HOME=/workspace-home",
+            "-v",
+            f"{LANG_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
             "-v",
             f"{AIRFLOW_ROOT_PATH}:/repo",
-            "-w",
-            go_example_ctr,
-            LANG_SDK_GO_BUILDER_IMAGE,
-            "go",
-            "tool",
-            "airflow-go-pack",
-            "--output",
-            f"{go_example_ctr}/bin/{LANG_SDK_GO_BUNDLE_NAME}",
-            ".",
-        ],
-        output=output,
-        check=True,
-    )
-    shutil.copy(LANG_SDK_GO_EXAMPLE_PATH / "bin" / LANG_SDK_GO_BUNDLE_NAME, go_dir / LANG_SDK_GO_BUNDLE_NAME)
-
-
-def _lang_sdk_build_java_jar(staging: Path, output: Output | None) -> None:
-    """Publish the Java SDK to mavenLocal then build the java_example jar into ``staging/java-artifacts``.
-
-    Runs in an ephemeral JDK container so the host needs no JDK; the Gradle distribution, dependency
-    and Maven caches persist between runs via the mounted dirs.
-    """
-    java_dir = staging / "java-artifacts"
-    java_dir.mkdir(parents=True, exist_ok=True)
-
-    uid_gid = f"{os.getuid()}:{os.getgid()}"
-    java_example_ctr = f"/repo/{LANG_SDK_JAVA_EXAMPLE_PATH.relative_to(AIRFLOW_ROOT_PATH).as_posix()}"
-
-    # java_example resolves the SDK plugin and libraries from mavenLocal(), so
-    # publish them first, then build the bundle with java-sdk's gradle wrapper
-    # pointed at the moved project (-p). --user keeps build outputs owned by the
-    # host user; HOME is set explicitly because that UID has no /etc/passwd entry;
-    # GRADLE_USER_HOME and the mounted ~/.m2 persist the Gradle distribution and
-    # dependency caches between runs.
-    LANG_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
-    java_docker_prefix = [
-        "docker",
-        "run",
-        "--rm",
-        "--user",
-        uid_gid,
-        "-e",
-        "GRADLE_USER_HOME=/repo/java-sdk/.gradle",
-        "-e",
-        "HOME=/workspace-home",
-        "-v",
-        f"{LANG_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
-        "-v",
-        f"{AIRFLOW_ROOT_PATH}:/repo",
-    ]
-    get_console(output=output).print("[info]Publishing Java SDK artifacts to local Maven repository")
-    run_command(
-        [
-            *java_docker_prefix,
-            "-w",
-            "/repo/java-sdk",
-            LANG_SDK_JAVA_BUILDER_IMAGE,
-            "./gradlew",
-            "publishToMavenLocal",
-            "-PskipSigning=true",
-            "--no-daemon",
-            "--console=plain",
-        ],
-        output=output,
-        check=True,
-    )
-    get_console(output=output).print(f"[info]Building Java jar in {LANG_SDK_JAVA_BUILDER_IMAGE}")
-    run_command(
-        [
-            *java_docker_prefix,
-            "-w",
-            "/repo/java-sdk",
-            LANG_SDK_JAVA_BUILDER_IMAGE,
-            "./gradlew",
-            "-p",
-            java_example_ctr,
-            "bundle",
-            "--no-daemon",
-            "--console=plain",
-        ],
-        output=output,
-        check=True,
-    )
+        ]
+        get_console(output=output).print("[info]Publishing Java SDK artifacts to local Maven repository")
+        run_command(
+            [
+                *java_docker_prefix,
+                "-w",
+                "/repo/java-sdk",
+                LANG_SDK_JAVA_BUILDER_IMAGE,
+                "./gradlew",
+                "publishToMavenLocal",
+                "-PskipSigning=true",
+                "--no-daemon",
+                "--console=plain",
+            ],
+            output=output,
+            check=True,
+        )
+        get_console(output=output).print(f"[info]Building Java jar in {LANG_SDK_JAVA_BUILDER_IMAGE}")
+        run_command(
+            [
+                *java_docker_prefix,
+                "-w",
+                "/repo/java-sdk",
+                LANG_SDK_JAVA_BUILDER_IMAGE,
+                "./gradlew",
+                "-p",
+                java_example_ctr,
+                "bundle",
+                "--no-daemon",
+                "--console=plain",
+            ],
+            output=output,
+            check=True,
+        )
     jars = list((LANG_SDK_JAVA_EXAMPLE_PATH / "build" / "bundle").glob("*.jar"))
     if not jars:
         get_console(output=output).print("[error]No jar produced by the Java bundle build")
@@ -2901,11 +2931,14 @@ def _setup_lang_sdk_test(
         # The worker-image build below produces this fixed tag; resolve it up-front so the config
         # rendering (which needs the tag, not the build result) does not depend on the parallel run.
         java_image = LANG_SDK_JAVA_WORKER_IMAGE
+    # In CI the Go/Java toolchains are provisioned + cached on the host (actions/setup-go, setup-java),
+    # so building the artifacts natively skips the toolchain-image pulls and reuses the runner caches.
+    native = os.environ.get("LANG_SDK_NATIVE_TOOLCHAIN", "").lower() == "true"
     with tempfile.TemporaryDirectory(prefix="lang_sdk_artifacts_") as tmp:
         staging = Path(tmp)
         steps: list[tuple[str, Callable[[Output | None], Any]]] = [
-            ("Build Go bundle", lambda o: _lang_sdk_build_go_bundle(staging, o)),
-            ("Build Java jar", lambda o: _lang_sdk_build_java_jar(staging, o)),
+            ("Build Go bundle", lambda o: _lang_sdk_build_go_bundle(staging, o, native=native)),
+            ("Build Java jar", lambda o: _lang_sdk_build_java_jar(staging, o, native=native)),
             ("Deploy localstack", lambda o: _lang_sdk_deploy_localstack(python, kubernetes_version, o)),
         ]
         if build_java_image:

dev/breeze/src/airflow_breeze/global_constants.py

@@ -229,6 +229,10 @@
 JAVA_SDK = "java"
 ALLOWED_SDKS = [JAVA_SDK]
 
+# JDK version used to build the Java SDK and its example bundles (e.g. the lang-SDK k8s system test).
+# Keep in sync with the toolchain the Java SDK Gradle build targets.
+JAVA_SDK_VERSION = "17"
+
 DEFAULT_ALLOWED_EXECUTOR = ALLOWED_EXECUTORS[0]
 ALLOWED_AUTH_MANAGERS = [SIMPLE_AUTH_MANAGER, FAB_AUTH_MANAGER]
 START_AIRFLOW_ALLOWED_EXECUTORS = [LOCAL_EXECUTOR, CELERY_EXECUTOR, EDGE_EXECUTOR]