Merge branch 'main' into refactor-container-lifecycle-test

Author: Miretpl

.github/actions/breeze/action.yml

@@ -41,6 +41,8 @@ runs:
     - name: "Install Breeze"
       shell: bash
       run: ./scripts/ci/install_breeze.sh
+      env:
+        PYTHON_VERSION: "${{ inputs.python-version }}"
     - name: "Free space"
       shell: bash
       run: breeze ci free-space

.github/actions/install-pre-commit/action.yml

@@ -21,10 +21,10 @@ description: 'Installs pre-commit and related packages'
 inputs:
   python-version:
     description: 'Python version to use'
-    default: "3.9"
+    default: "3.10"
   uv-version:
     description: 'uv version to use'
-    default: "0.7.14"  # Keep this comment to allow automatic replacement of uv version
+    default: "0.7.19"  # Keep this comment to allow automatic replacement of uv version
   pre-commit-version:
     description: 'pre-commit version to use'
     default: "4.2.0"  # Keep this comment to allow automatic replacement of pre-commit version

.github/actions/migration_tests/action.yml

@@ -24,9 +24,9 @@ runs:
     - name: "Test migration file 2 to 3 migration: ${{env.BACKEND}}"
       shell: bash
       run: |
-        breeze shell "${{ env.AIRFLOW_2_CMD }}" --use-airflow-version 2.11.0 --answer y &&
-        breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANGERS}}
-                    ${{ env.AIRFLOW_3_CMD }}" --no-db-cleanup
+        breeze shell "${AIRFLOW_2_CMD}" --use-airflow-version 2.11.0 --answer y &&
+        breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${DB_MANGERS}
+                    ${AIRFLOW_3_CMD}" --no-db-cleanup
       env:
         COMPOSE_PROJECT_NAME: "docker-compose"
         DB_RESET: "false"
@@ -47,9 +47,9 @@ runs:
     - name: "Test ORM migration 2 to 3: ${{env.BACKEND}}"
       shell: bash
       run: >
-        breeze shell "${{ env.AIRFLOW_2_CMD }}" --use-airflow-version 2.11.0 --answer y &&
-        breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANGERS}}
-             ${{ env.AIRFLOW_3_CMD }}" --no-db-cleanup
+        breeze shell "${AIRFLOW_2_CMD}" --use-airflow-version 2.11.0 --answer y &&
+        breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${DB_MANGERS}
+             ${AIRFLOW_3_CMD}" --no-db-cleanup
       env:
         COMPOSE_PROJECT_NAME: "docker-compose"
         DB_RESET: "false"
@@ -69,7 +69,7 @@ runs:
     - name: "Test ORM migration ${{env.BACKEND}}"
       shell: bash
       run: >
-        breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANAGERS}} &&
+        breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${DB_MANAGERS} &&
         airflow db reset -y &&
         airflow db migrate --to-revision heads &&
         airflow db downgrade -n 2.7.0 -y &&
@@ -86,7 +86,7 @@ runs:
       shell: bash
       run: >
         breeze shell
-        "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANAGERS}} &&
+        "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${DB_MANAGERS} &&
         airflow db reset -y &&
         airflow db downgrade -n 2.7.0 -y &&
         airflow db migrate -s"

.github/actions/prepare_all_ci_images/action.yml

@@ -36,6 +36,7 @@ runs:
     #       this should be implemented in stash action as list of keys to download.
     #       That includes 3.9 - 3.12 as we are backporting it to v3-0-test branch
     #       This is captured in https://github.com/apache/airflow/issues/45268
+    #       So we actually need 3.9 even if 3.9 support on main is dropped!
     - name: "Restore CI docker image ${{ inputs.platform }}:3.9"
       uses: ./.github/actions/prepare_single_ci_image
       with:

.github/workflows/additional-ci-image-checks.yml

@@ -122,8 +122,8 @@ jobs:
     runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       UPGRADE_TO_NEWER_DEPENDENCIES: false
-      PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
-      PYTHON_VERSION: ${{ inputs.default-python-version }}
+      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+      PYTHON_VERSION: "${{ inputs.default-python-version }}"
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}

.github/workflows/additional-prod-image-tests.yml

@@ -70,7 +70,7 @@ jobs:
       runners: ${{ inputs.runners }}
       platform: ${{ inputs.platform }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
-      default-python-version: ${{ inputs.default-python-version }}
+      default-python-version: "${{ inputs.default-python-version }}"
       branch: ${{ inputs.default-branch }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
       constraints-branch: ${{ inputs.constraints-branch }}
@@ -85,7 +85,7 @@ jobs:
       runners: ${{ inputs.runners }}
       platform: ${{ inputs.platform }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
-      default-python-version: ${{ inputs.default-python-version }}
+      default-python-version: "${{ inputs.default-python-version }}"
       branch: ${{ inputs.default-branch }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
       constraints-branch: ${{ inputs.constraints-branch }}
@@ -116,13 +116,13 @@ jobs:
         with:
           platform: ${{ inputs.platform }}
           image-type: "prod"
-          python: ${{ inputs.default-python-version }}
+          python: "${{ inputs.default-python-version }}"
           use-uv: ${{ inputs.use-uv }}
       - name: "Test examples of PROD image building"
         env:
           GITHUB_REPOSITORY: ${{ github.repository }}
           DEFAULT_BRANCH: ${{ inputs.default-branch }}
-          DEFAULT_PYTHON_VERSION: ${{ inputs.default-python-version }}
+          DEFAULT_PYTHON_VERSION: "${{ inputs.default-python-version }}"
         run: "
           cd ./docker-tests && \
           TEST_IMAGE=\"ghcr.io/$GITHUB_REPOSITORY/$DEFAULT_BRANCH\

.github/workflows/airflow-distributions-tests.yml

@@ -61,11 +61,16 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether local venv should be used for tests (true/false)"
         required: true
         type: string
+      test-timeout:
+        required: false
+        type: number
+        default: 60
+
 permissions:
   contents: read
 jobs:
   distributions-tests:
-    timeout-minutes: 80
+    timeout-minutes: ${{ fromJSON(inputs.test-timeout) }}
     name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests
     runs-on: ${{ fromJSON(inputs.runners) }}
     strategy:

.github/workflows/basic-tests.yml

@@ -223,7 +223,7 @@ jobs:
           VERBOSE: "false"
           SKIP_BREEZE_PRE_COMMITS: "true"
           SKIP: ${{ inputs.skip-pre-commits }}
-          COLUMNS: "250"
+          COLUMNS: "202"
 
   test-git-clone-on-windows:
     timeout-minutes: 5
@@ -267,6 +267,8 @@ jobs:
           skip-pre-commits: ${{ inputs.skip-pre-commits }}
       - name: "Autoupdate all pre-commits"
         run: pre-commit autoupdate
+      - name: "Autoupdate Lucas-C/pre-commit-hooks to bleeding edge"
+        run: pre-commit autoupdate --bleeding-edge --freeze --repo https://github.com/Lucas-C/pre-commit-hooks
       - name: "Run automated upgrade for black"
         run: >
           pre-commit run
@@ -284,25 +286,23 @@ jobs:
       # For UV we are not failing the upgrade installers check if it is updated because
       # it is upgraded very frequently, so we want to manually upgrade it rather than
       # get notified about it - until it stabilizes in 1.* version
-      - name: "Run automated upgrade for uv (open to see if new version is updated)"
+      - name: "Run automated upgrade for uv (not failing - just informational)"
         run: >
           pre-commit run
           --all-files --show-diff-on-failure --color always --verbose
           --hook-stage manual update-installers-and-pre-commit || true
         if: always()
         env:
-          UPGRADE_UV: "true"
           UPGRADE_PIP: "false"
           UPGRADE_PRE_COMMIT: "false"
           UPGRADE_NODE_LTS: "false"
-      - name: "Run automated upgrade for pip, pre-commit and node"
+      - name: "Run automated upgrade for pip, pre-commit and node (failing if needed)"
         run: >
           pre-commit run
           --all-files --show-diff-on-failure --color always --verbose
           --hook-stage manual update-installers-and-pre-commit
         if: always()
         env:
-          UPGRADE_UV: "false"
           UPGRADE_PIP: "true"
           UPGRADE_PRE_COMMIT: "true"
           UPGRADE_NODE_LTS: "true"

.github/workflows/ci-amd.yml

@@ -186,7 +186,7 @@ jobs:
       - name: Install pre-commit, uv, and pre-commit-uv
         shell: bash
         env:
-          UV_VERSION: "0.7.14"  # Keep this comment to allow automatic replacement of uv version
+          UV_VERSION: "0.7.19"  # Keep this comment to allow automatic replacement of uv version
           PRE_COMMIT_VERSION: "4.2.0"  # Keep this comment to allow automatic replacement of pre-commit version
           PRE_COMMIT_UV_VERSION: "4.1.4"  # Keep this comment to allow automatic replacement of pre-commit-uv version
         run: |
@@ -209,7 +209,7 @@ jobs:
       run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}}
       run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
       needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
-      default-python-version: ${{needs.build-info.outputs.default-python-version}}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
       skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}}
       canary-run: ${{needs.build-info.outputs.canary-run}}
@@ -248,14 +248,13 @@ jobs:
       contents: read
       packages: write
       id-token: write
-    if: needs.build-info.outputs.canary-run == 'true'
     with:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
@@ -275,6 +274,8 @@ jobs:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
+      generate-pypi-constraints: "true"
       # generate no providers constraints only in canary builds - they take quite some time to generate
       # they are not needed for regular builds, they are only needed to update constraints in canaries
       generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }}
@@ -296,7 +297,7 @@ jobs:
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
       latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
       basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
@@ -329,7 +330,7 @@ jobs:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
       canary-run: ${{ needs.build-info.outputs.canary-run }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }}
       # yamllint disable rule:line-length
@@ -352,7 +353,7 @@ jobs:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
       helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       use-uv: ${{ needs.build-info.outputs.use-uv }}
     if: >
       needs.build-info.outputs.needs-helm-tests == 'true' &&
@@ -612,7 +613,7 @@ jobs:
       providers-test-types-list-as-strings-in-json: >
         ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
       excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
@@ -636,7 +637,7 @@ jobs:
       testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }}
       testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }}
       run-system-tests: ${{ needs.build-info.outputs.run-tests }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
       default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
@@ -721,7 +722,7 @@ jobs:
       upload-image-artifact: "true"
       upload-package-artifact: "true"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       branch: ${{ needs.build-info.outputs.default-branch }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
@@ -742,7 +743,7 @@ jobs:
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
       disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       canary-run: ${{ needs.build-info.outputs.canary-run }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
     if: needs.build-info.outputs.prod-image-build == 'true'
@@ -776,14 +777,15 @@ jobs:
     with:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
       distribution-name: "task-sdk"
       distribution-cmd-format: "prepare-task-sdk-distributions"
       test-type: "task-sdk-tests"
       use-local-venv: 'false'
+      test-timeout: 20
     if: >
       ( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
       needs.build-info.outputs.run-tests == 'true' &&
@@ -819,12 +821,19 @@ jobs:
           go-version: 1.24
           cache-dependency-path: go-sdk/go.sum
 
+      # keep this in sync with go.mod in go-sdk/
+      - name: Setup Gotestsum
+        shell: bash
+        run: |
+          go install gotest.tools/gotestsum@ddd0b05a6878e2e8257a2abe6e7df66cebc53d0e  # v1.12.3
+          gotestsum --version
+
       - name: "Cleanup dist files"
         run: rm -fv ./dist/*
 
       - name: Run Go tests
         working-directory: ./go-sdk
-        run: go test -v ./...
+        run: gotestsum --format testname ./...
 
   tests-airflow-ctl:
     name: "Airflow CTL tests"
@@ -836,14 +845,15 @@ jobs:
     with:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
       distribution-name: "airflow-ctl"
       distribution-cmd-format: "prepare-airflow-ctl-distributions"
       test-type: "airflow-ctl-tests"
       use-local-venv: 'true'
+      test-timeout: 20
     if: >
       ( needs.build-info.outputs.run-airflow-ctl-tests == 'true' ||
       needs.build-info.outputs.run-tests == 'true' &&
@@ -889,7 +899,7 @@ jobs:
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
-      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
@@ -948,7 +958,7 @@ jobs:
       - name: "Setup python"
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
         with:
-          python-version: ${{ inputs.default-python-version }}
+          python-version: "${{ inputs.default-python-version }}"
       - name: "Summarize all warnings"
         run: |
           ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \