Auto-upgrade sphinx-airflow-theme version in prek hook (#63763)

Add sphinx-airflow-theme to the upgrade-important-versions prek hook so
it automatically fetches the latest version from
https://airflow.apache.org/sphinx-airflow-theme/LATEST_VERSION.txt and
updates the whl URL in devel-common/pyproject.toml.

Also fix selective checks tests (add update-uv-lock to skip-prek-hooks
expectations) and resolve mypy errors in selective_checks.py.

Author: potiuk

.github/actions/install-prek/action.yml

@@ -27,7 +27,7 @@ inputs:
     default: "0.10.10"  # Keep this comment to allow automatic replacement of uv version
   prek-version:
     description: 'prek version to use'
-    default: "0.3.5"  # Keep this comment to allow automatic replacement of prek version
+    default: "0.3.6"  # Keep this comment to allow automatic replacement of prek version
   save-cache:
     description: "Whether to save prek cache"
     required: true

.pre-commit-config.yaml

@@ -259,24 +259,6 @@ repos:
           ^scripts/ci/prek/update_installers_and_prek\.py$
         pass_filenames: false
         require_serial: true
-      - id: uv-sync
-        name: Run uv sync
-        entry: uv sync
-        language: system
-        files: (^|/)pyproject\.toml$
-        pass_filenames: false
-        require_serial: true
-      - id: update-uv-lock
-        name: Update uv.lock (manual)
-        entry: ./scripts/ci/prek/update_uv_lock.py
-        stages: ['manual']
-        language: python
-        files: >
-          (?x)
-          ^pyproject\.toml$|
-          ^uv\.lock$
-        pass_filenames: false
-        require_serial: true
   - repo: https://github.com/adamchainz/blacken-docs
     rev: fda77690955e9b63c6687d8806bafd56a526e45f  # frozen: 1.20.0
     hooks:
@@ -998,8 +980,22 @@ repos:
         pass_filenames: false
         files: ^dev/breeze/src/airflow_breeze/global_constants\.py$
         require_serial: true
+        # This is a fast regular hook that runs when any pyproject.toml changes
+        # It runs locally and usually will not result in modifying the lock unnecessarily
+        # Unless there is a conflict and uv will determine that the lock needs to be updated to resolve it
+      - id: update-uv-lock
+        name: Update uv.lock
+        entry: uv sync
+        language: system
+        files: >
+          (?x)
+          (^|/)pyproject\.toml$|
+          ^uv\.lock$
+        pass_filenames: false
+        require_serial: true
         ## ADD MOST PREK HOOK ABOVE THAT LINE
         # The below prek hooks are those requiring CI image to be built
+        ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
       - id: mypy-dev
         stages: ['pre-push']
         name: Run mypy for dev
@@ -1063,10 +1059,23 @@ repos:
           ^airflow-core/src/airflow/api_fastapi/core_api/datamodels/.*\.py$|
           ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/(datamodels|routes|services|openapi)/.*\.py$
         require_serial: true
-        ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
       - id: check-contextmanager-class-decorators
         name: Check for problematic context manager class decorators
         entry: ./scripts/ci/prek/check_contextmanager_class_decorators.py
         language: python
         files: .*test.*\.py$
         pass_filenames: true
+        # This is a manual hook, run by `breeze ci upgrade` - upgrading all dependencies inside the
+        # Breeze CI image - which allows checking all dependencies for all providers.
+        # ALWAYS keep it at the end so that it can take into account all the other hook's changes.
+      - id: update-uv-lock
+        stages: ['manual']
+        name: Update uv.lock (manual)
+        entry: breeze run uv lock --upgrade
+        language: system
+        files: >
+          (?x)
+          (^|/)pyproject\.toml$|
+          ^uv\.lock$
+        pass_filenames: false
+        require_serial: true

Dockerfile.ci

@@ -1749,7 +1749,7 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe
 ARG AIRFLOW_PIP_VERSION=26.0.1
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
 ARG AIRFLOW_UV_VERSION=0.10.10
-ARG AIRFLOW_PREK_VERSION="0.3.5"
+ARG AIRFLOW_PREK_VERSION="0.3.6"
 
 # UV_LINK_MODE=copy is needed since we are using cache mounted from the host
 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \

contributing-docs/07_local_virtualenv.rst

@@ -246,7 +246,7 @@ in editable mode.
 The dependencies for providers are configured in ``providers/PROVIDER/pyproject.toml`` files -
 separately for each provider. You can find there two types of ``dependencies`` - production runtime
 dependencies, and sometimes ``development dependencies`` (in ``dev`` dependency group) which are needed
-to run tests and are installed automatically when you install environment with ``uv-sync``.
+to run tests and are installed automatically when you install environment with ``uv sync``.
 
 If you want to add another dependency to a provider, you should add it to corresponding ``pyproject.toml``,
 add the files to your commit with ``git add`` and run ``prek`` to update generated dependencies.

contributing-docs/testing/unit_tests.rst

@@ -1259,7 +1259,7 @@ or run the ``--force-lowest-dependencies`` switch directly from the breeze comma
     breeze shell --force-lowest-dependencies --test-type "Core"
 
 
-The way it works: after you enter the breeze container, you run ``uv-sync`` in the airflow-core
+The way it works: after you enter the breeze container, you run ``uv sync`` in the airflow-core
 folder to downgrade the dependencies to the lowest version that is compatible
 with the dependencies specified in airflow-core dependencies. You will see it in the output of the breeze
 command as a sequence of downgrades like this:

dev/breeze/doc/ci/02_images.md

@@ -444,7 +444,7 @@ can be used for CI images:
 | `ADDITIONAL_DEV_APT_ENV`          |                             | Additional env variables defined when installing dev deps                                                         |
 | `AIRFLOW_PIP_VERSION`             | `26.0.1`                    | `pip` version used.                                                                                               |
 | `AIRFLOW_UV_VERSION`              | `0.10.10`                   | `uv` version used.                                                                                                |
-| `AIRFLOW_PREK_VERSION`            | `0.3.5`                     | `prek` version used.                                                                                              |
+| `AIRFLOW_PREK_VERSION`            | `0.3.6`                     | `prek` version used.                                                                                              |
 | `AIRFLOW_USE_UV`                  | `true`                      | Whether to use UV for installation.                                                                               |
 | `PIP_PROGRESS_BAR`                | `on`                        | Progress bar for PIP installation                                                                                 |
 

dev/breeze/pyproject.toml

@@ -57,7 +57,7 @@ dependencies = [
     "jinja2>=3.1.5",
     "jsonschema>=4.19.1",
     "packaging>=25.0",
-    "prek>=0.3.5",
+    "prek>=0.3.6",
     "psutil>=5.9.6",
     "pygithub>=2.1.1",
     "pytest-xdist>=3.3.1",

dev/breeze/src/airflow_breeze/commands/ci_commands.py

@@ -794,7 +794,7 @@ def upgrade(
         ),
         (
             "update-uv-lock",
-            "prek --all-files --show-diff-on-failure --color always --verbose --stage manual update-uv-lock",
+            "prek --all-files --show-diff-on-failure --color always --verbose update-uv-lock --stage manual",
         ),
     ]
     step_enabled = {

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -264,7 +264,7 @@ class VersionedFile(NamedTuple):
 AIRFLOW_USE_UV = False
 GITPYTHON_VERSION = "3.1.46"
 RICH_VERSION = "14.3.3"
-PREK_VERSION = "0.3.5"
+PREK_VERSION = "0.3.6"
 HATCH_VERSION = "1.16.5"
 PYYAML_VERSION = "6.0.3"
 

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -57,6 +57,7 @@
     SelectiveCoreTestType,
     SelectiveProvidersTestType,
     SelectiveTaskSdkTestType,
+    SelectiveTestType,
     all_helm_test_packages,
     all_selective_core_test_types,
     providers_test_type,
@@ -145,7 +146,7 @@ class AllProvidersSentinel:
 
 ALL_PROVIDERS_SENTINEL = AllProvidersSentinel()
 
-T = TypeVar("T", FileGroupForCi, SelectiveCoreTestType)
+T = TypeVar("T", FileGroupForCi, SelectiveTestType)
 
 
 class HashableDict(dict[T, list[str]]):
@@ -351,7 +352,7 @@ def __hash__(self):
     r"^providers/tests/standard/operators/test_python.py",
 ]
 
-TEST_TYPE_MATCHES: HashableDict[SelectiveCoreTestType] = HashableDict(
+TEST_TYPE_MATCHES: HashableDict[SelectiveTestType] = HashableDict(
     {
         SelectiveCoreTestType.API: [
             r"^airflow-core/src/airflow/api/",
@@ -423,7 +424,7 @@ def _exclude_files_with_regexps(files: tuple[str, ...], matched_files, exclude_r
 
 @clearable_cache
 def _matching_files(
-    files: tuple[str, ...], match_group: FileGroupForCi, match_dict: HashableDict
+    files: tuple[str, ...], match_group: FileGroupForCi | SelectiveTestType, match_dict: HashableDict
 ) -> list[str]:
     matched_files: list[str] = []
     match_regexps = match_dict[match_group]
@@ -819,7 +820,9 @@ def kubernetes_combos(self) -> list[str]:
     def kubernetes_combos_list_as_string(self) -> str:
         return " ".join(self.kubernetes_combos)
 
-    def _matching_files(self, match_group: FileGroupForCi, match_dict: HashableDict) -> list[str]:
+    def _matching_files(
+        self, match_group: FileGroupForCi | SelectiveTestType, match_dict: HashableDict
+    ) -> list[str]:
         return _matching_files(self._files, match_group, match_dict)
 
     def _should_be_run(self, source_area: FileGroupForCi) -> bool:
@@ -1027,9 +1030,7 @@ def prod_image_build(self) -> bool:
             or self.run_ui_e2e_tests
         )
 
-    def _select_test_type_if_matching(
-        self, test_types: set[str], test_type: SelectiveCoreTestType
-    ) -> list[str]:
+    def _select_test_type_if_matching(self, test_types: set[str], test_type: SelectiveTestType) -> list[str]:
         matched_files = self._matching_files(test_type, TEST_TYPE_MATCHES)
         count = len(matched_files)
         if count > 0:
@@ -1308,7 +1309,7 @@ def pyproject_toml_changed(self) -> bool:
         try:
             import tomllib
         except ImportError:
-            import tomli as tomllib
+            import tomli as tomllib  # type: ignore[no-redef]
 
         self._new_toml = tomllib.loads(new_result.stdout)
         self._old_toml = tomllib.loads(old_result.stdout)
@@ -1382,6 +1383,7 @@ def docs_list_as_string(self) -> str | None:
     def skip_prek_hooks(self) -> str:
         prek_hooks_to_skip = set()
         prek_hooks_to_skip.add("identity")
+        prek_hooks_to_skip.add("update-uv-lock")
         if self._default_branch != "main":
             # Skip those tests on all "release" branches
             prek_hooks_to_skip.update(
@@ -1480,7 +1482,7 @@ def selected_providers_list_as_string(self) -> str | None:
         return " ".join(sorted(p for p in affected_providers if p not in suspended))
 
     def get_job_label(self, event_type: str, branch: str):
-        import requests
+        import requests  # type: ignore[import-untyped]
 
         job_name = "Basic tests"
         workflow_name = "ci-amd-arm.yml"
@@ -1728,7 +1730,7 @@ def provider_dependency_bump(self) -> bool:
         try:
             import tomllib
         except ImportError:
-            import tomli as tomllib
+            import tomli as tomllib  # type: ignore[no-redef]
 
         violations = []
         for pyproject_file in pyproject_files: