From be57acb6c7b680ae75a69f62e86483e60d963b57 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 4 Feb 2026 01:11:44 +0100 Subject: [PATCH 01/10] Make Mypy plugins installable --- dev/mypy/.gitignore | 9 + dev/mypy/.pre-commit-config.yaml | 44 ++++ dev/mypy/LICENSE | 201 ++++++++++++++++++ dev/mypy/NOTICE | 5 + dev/mypy/README.md | 65 ++++++ dev/mypy/RELEASE_NOTES.rst | 32 +++ dev/mypy/newsfragments/config.toml | 50 +++++ dev/mypy/pyproject.toml | 122 +++++++++++ .../src/airflow_mypy}/__init__.py | 5 + .../src/airflow_mypy/plugins}/__init__.py | 0 .../src/airflow_mypy/plugins}/decorators.py | 0 .../src/airflow_mypy/plugins}/outputs.py | 0 dev/mypy/tests/test_plugins.py | 52 +++++ dev/pyproject.toml | 1 + pyproject.toml | 6 +- 15 files changed, 590 insertions(+), 2 deletions(-) create mode 100644 dev/mypy/.gitignore create mode 100644 dev/mypy/.pre-commit-config.yaml create mode 100644 dev/mypy/LICENSE create mode 100644 dev/mypy/NOTICE create mode 100644 dev/mypy/README.md create mode 100644 dev/mypy/RELEASE_NOTES.rst create mode 100644 dev/mypy/newsfragments/config.toml create mode 100644 dev/mypy/pyproject.toml rename dev/{airflow_mypy/plugin => mypy/src/airflow_mypy}/__init__.py (86%) rename dev/{airflow_mypy => mypy/src/airflow_mypy/plugins}/__init__.py (100%) rename dev/{airflow_mypy/plugin => mypy/src/airflow_mypy/plugins}/decorators.py (100%) rename dev/{airflow_mypy/plugin => mypy/src/airflow_mypy/plugins}/outputs.py (100%) create mode 100644 dev/mypy/tests/test_plugins.py diff --git a/dev/mypy/.gitignore b/dev/mypy/.gitignore new file mode 100644 index 0000000000000..4bee7fc36df78 --- /dev/null +++ b/dev/mypy/.gitignore @@ -0,0 +1,9 @@ +# Build artifacts +dist/ +build/ +*.egg-info/ + +# Python cache +__pycache__/ +*.py[cod] +*$py.class diff --git a/dev/mypy/.pre-commit-config.yaml b/dev/mypy/.pre-commit-config.yaml new file mode 100644 index 0000000000000..a75260a92762d --- /dev/null +++ b/dev/mypy/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.2.0' +default_language_version: + python: python3 + node: 22.19.0 + golang: 1.24.0 +repos: + - repo: local + hooks: + - id: mypy-airflow-mypy + stages: ['pre-push'] + name: Run mypy for airflow-mypy plugins + language: python + entry: ../../scripts/ci/prek/mypy.py + files: + (?x) + ^src/airflow_mypy/.*\.py$| + ^tests/.*\.py$ + require_serial: true + - id: mypy-airflow-mypy-manual + stages: ['manual'] + name: Run mypy for airflow-mypy (manual) + language: python + entry: ../../scripts/ci/prek/mypy_folder.py dev/mypy + pass_filenames: false + files: ^.*\.py$ + require_serial: true diff --git a/dev/mypy/LICENSE b/dev/mypy/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/dev/mypy/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/dev/mypy/NOTICE b/dev/mypy/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/dev/mypy/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/dev/mypy/README.md b/dev/mypy/README.md new file mode 100644 index 0000000000000..5f3b7e92b6127 --- /dev/null +++ b/dev/mypy/README.md @@ -0,0 +1,65 @@ + + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Apache Airflow Mypy Plugins](#apache-airflow-mypy-plugins) + - [Installation](#installation) + - [Usage](#usage) + - [Available Plugins](#available-plugins) + +# Apache Airflow Mypy Plugins + +This package provides Mypy plugins for Apache Airflow to enhance type checking capabilities. + +## Installation + +```bash +pip install apache-airflow-mypy +``` + +## Usage + +Add the plugins to your `mypy.ini` or `pyproject.toml` configuration: + +### Using mypy.ini + +```ini +[mypy] +plugins = airflow_mypy.plugins.decorators, airflow_mypy.plugins.outputs +``` + +### Using pyproject.toml + +```toml +[tool.mypy] +plugins = ["airflow_mypy.plugins.decorators", "airflow_mypy.plugins.outputs"] +``` + +## Available Plugins + +### decorators Plugin + +Provides type checking support for Airflow decorators that modify function signatures. + +### outputs Plugin + +Handles type checking for operator outputs and XComArg types, allowing proper type inference when passing task outputs between tasks. diff --git a/dev/mypy/RELEASE_NOTES.rst b/dev/mypy/RELEASE_NOTES.rst new file mode 100644 index 0000000000000..868ad802db55d --- /dev/null +++ b/dev/mypy/RELEASE_NOTES.rst @@ -0,0 +1,32 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +apache-airflow-mypy 0.1.0 +------------------------------- + +Initial release of apache-airflow-mypy, a package providing Mypy plugins for Apache Airflow. + +Features +^^^^^^^^ + +- Mypy plugin for typed decorators that modify function signatures +- Mypy plugin for operator outputs and XComArg type handling +- Support for type checking Airflow decorators with injected parameters +- Proper type inference when passing task outputs between tasks + +This package makes the Airflow Mypy plugins installable and reusable for users who want to +enhance type checking in their Airflow DAGs. diff --git a/dev/mypy/newsfragments/config.toml b/dev/mypy/newsfragments/config.toml new file mode 100644 index 0000000000000..41888e8f06e31 --- /dev/null +++ b/dev/mypy/newsfragments/config.toml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +[tool.towncrier] +name = "Apache Airflow Mypy" +filename = "RELEASE_NOTES.rst" +underlines = ["-", '^'] + +[[tool.towncrier.type]] +directory = "significant" +name = "Significant Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "improvement" +name = "Improvements" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bug Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Doc only Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = true diff --git a/dev/mypy/pyproject.toml b/dev/mypy/pyproject.toml new file mode 100644 index 0000000000000..86cfa85986aa7 --- /dev/null +++ b/dev/mypy/pyproject.toml @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +[project] +name = "apache-airflow-mypy" +dynamic = ["version"] +description = "Apache Airflow Mypy plugins" +readme = { file = "README.md", content-type = "text/markdown" } +license = "Apache-2.0" +license-files = ["LICENSE", "NOTICE"] +requires-python = ">=3.10,!=3.14" + +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow", "orchestration", "workflow", "dag", "pipelines", "automation", "data" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: System :: Monitoring", +] +dependencies = [ + "mypy>=1.0.0" +] + +[project.urls] +"Bug Tracker" = "https://github.com/apache/airflow/issues" +Documentation = "https://airflow.apache.org/docs/apache-airflow-mypy/stable/index.html" +Downloads = "https://archive.apache.org/dist/airflow/airflow-mypy/" +Homepage = "https://airflow.apache.org/" +"Release Notes" = "https://airflow.apache.org/docs/apache-airflow-mypy/stable/changelog.html" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Source Code" = "https://github.com/apache/airflow" +LinkedIn = "https://www.linkedin.com/company/apache-airflow/" +Mastodon = "https://fosstodon.org/@airflow" +Bluesky = "https://bsky.app/profile/apache-airflow.bsky.social" +YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.version] +path = "src/airflow_mypy/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/airflow_mypy"] + +[tool.ruff] +extend = "../../pyproject.toml" +src = ["src"] +namespace-packages = ["src/airflow_mypy"] + +[tool.ruff.lint.extend-per-file-ignores] +"tests/*" = ["S101", "TRY002"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] + +[tool.towncrier] +name = "Apache Airflow Mypy" +filename = "RELEASE_NOTES.rst" +directory = "newsfragments" +underlines = ["-", "^"] + +[[tool.towncrier.type]] +directory = "significant" +name = "Significant Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "improvement" +name = "Improvements" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bug Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Doc only Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = true + +[tool.uv] +required-version = ">=0.6.3" diff --git a/dev/airflow_mypy/plugin/__init__.py b/dev/mypy/src/airflow_mypy/__init__.py similarity index 86% rename from dev/airflow_mypy/plugin/__init__.py rename to dev/mypy/src/airflow_mypy/__init__.py index 217e5db960782..32a64efabc5c0 100644 --- a/dev/airflow_mypy/plugin/__init__.py +++ b/dev/mypy/src/airflow_mypy/__init__.py @@ -15,3 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + +__path__ = __import__("pkgutil").extend_path(__path__, __name__) + +__version__ = "0.1.0" diff --git a/dev/airflow_mypy/__init__.py b/dev/mypy/src/airflow_mypy/plugins/__init__.py similarity index 100% rename from dev/airflow_mypy/__init__.py rename to dev/mypy/src/airflow_mypy/plugins/__init__.py diff --git a/dev/airflow_mypy/plugin/decorators.py b/dev/mypy/src/airflow_mypy/plugins/decorators.py similarity index 100% rename from dev/airflow_mypy/plugin/decorators.py rename to dev/mypy/src/airflow_mypy/plugins/decorators.py diff --git a/dev/airflow_mypy/plugin/outputs.py b/dev/mypy/src/airflow_mypy/plugins/outputs.py similarity index 100% rename from dev/airflow_mypy/plugin/outputs.py rename to dev/mypy/src/airflow_mypy/plugins/outputs.py diff --git a/dev/mypy/tests/test_plugins.py b/dev/mypy/tests/test_plugins.py new file mode 100644 index 0000000000000..dc48ca41bc9ad --- /dev/null +++ b/dev/mypy/tests/test_plugins.py @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +def test_decorators_plugin_import(): + """Test that decorators plugin can be imported.""" + from airflow_mypy.plugins import decorators + + assert hasattr(decorators, "plugin") + assert callable(decorators.plugin) + + +def test_outputs_plugin_import(): + """Test that outputs plugin can be imported.""" + from airflow_mypy.plugins import outputs + + assert hasattr(outputs, "plugin") + assert callable(outputs.plugin) + + +def test_decorators_plugin_entrypoint(): + """Test that decorators plugin entry point works.""" + from airflow_mypy.plugins.decorators import plugin + + # The plugin function should return a plugin class when called with a version string + plugin_class = plugin("1.0") + assert plugin_class is not None + + +def test_outputs_plugin_entrypoint(): + """Test that outputs plugin entry point works.""" + from airflow_mypy.plugins.outputs import plugin + + # The plugin function should return a plugin class when called with a version string + plugin_class = plugin("1.0") + assert plugin_class is not None diff --git a/dev/pyproject.toml b/dev/pyproject.toml index 7cb9ae40faf81..4f185d506f43d 100644 --- a/dev/pyproject.toml +++ b/dev/pyproject.toml @@ -43,6 +43,7 @@ maintainers = [ version = "0.0.1" dependencies = [ + "apache-airflow-mypy", "beautifulsoup4>=4.12.0", "click>=8.3.0", "icalendar>=5.0.0", diff --git a/pyproject.toml b/pyproject.toml index 130878aef6b1b..ca01af708bc41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1024,8 +1024,8 @@ no_implicit_optional = true warn_redundant_casts = true warn_unused_ignores = false plugins = [ - "dev/airflow_mypy/plugin/decorators.py", - "dev/airflow_mypy/plugin/outputs.py", + "airflow_mypy.plugins.decorators", + "airflow_mypy.plugins.outputs", ] pretty = true show_error_codes = true @@ -1634,6 +1634,7 @@ apache-aurflow-docker-stack = false apache-airflow = {workspace = true} apache-airflow-breeze = {workspace = true} apache-airflow-dev = {workspace = true} +apache-airflow-mypy = {workspace = true} apache-airflow-scripts = {workspace = true} apache-airflow-core = {workspace = true} apache-airflow-ctl = {workspace = true} @@ -1768,6 +1769,7 @@ members = [ "airflow-core", "airflow-e2e-tests", "dev/breeze", + "dev/mypy", "dev/registry", "airflow-ctl", "airflow-ctl-tests", From 17a7f4af5dfb531a8bbbfc2d226d815329c89ebe Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sun, 22 Mar 2026 18:33:48 +0100 Subject: [PATCH 02/10] Add Breeze release command for Mypy plugins --- dev/README_RELEASE_MYPY.md | 241 ++++++++++++++++++ .../doc/09_release_management_tasks.rst | 26 ++ .../commands/release_management_commands.py | 33 +++ .../src/airflow_breeze/utils/path_utils.py | 4 + 4 files changed, 304 insertions(+) create mode 100644 dev/README_RELEASE_MYPY.md diff --git a/dev/README_RELEASE_MYPY.md b/dev/README_RELEASE_MYPY.md new file mode 100644 index 0000000000000..d51d808889ba6 --- /dev/null +++ b/dev/README_RELEASE_MYPY.md @@ -0,0 +1,241 @@ + + + + +**Table of contents** + +- [What the apache-airflow-mypy distribution is](#what-the-apache-airflow-mypy-distribution-is) +- [Decide when to release](#decide-when-to-release) +- [Versioning](#versioning) +- [Prepare Regular apache-airflow-mypy distributions (RC)](#prepare-regular-apache-airflow-mypy-distributions-rc) + - [Generate release notes](#generate-release-notes) + - [Build apache-airflow-mypy distributions for SVN apache upload](#build-apache-airflow-mypy-distributions-for-svn-apache-upload) + - [Build and sign the source and convenience packages](#build-and-sign-the-source-and-convenience-packages) + - [Add tags in git](#add-tags-in-git) + - [Commit the source packages to Apache SVN repo](#commit-the-source-packages-to-apache-svn-repo) + - [Publish the distributions to PyPI (release candidates)](#publish-the-distributions-to-pypi-release-candidates) + - [Prepare voting email](#prepare-voting-email) + - [Verify the release candidate](#verify-the-release-candidate) +- [Publish release](#publish-release) + - [Summarize the voting](#summarize-the-voting) + - [Publish release to SVN](#publish-release-to-svn) + - [Publish the packages to PyPI](#publish-the-packages-to-pypi) + - [Add tags in git](#add-tags-in-git-1) + - [Notify developers of release](#notify-developers-of-release) + + + +------------------------------------------------------------------------------------------------------------ + +# What the apache-airflow-mypy distribution is + +The `apache-airflow-mypy` package provides Mypy plugins for Apache Airflow to enhance type checking capabilities. +It includes plugins for typed decorators and operator output type handling. + +The Release Manager prepares `apache-airflow-mypy` packages separately from the main Airflow Release, using +`breeze` commands and accompanying scripts. This document provides an overview of the command line tools +needed to prepare the packages. + +# Decide when to release + +You can release `apache-airflow-mypy` distributions separately from the main Airflow on an ad-hoc basis, +whenever we find that the mypy plugins need to be released - due to new features, bug fixes, or improvements +to type checking support. + +# Versioning + +We are using the [SEMVER](https://semver.org/) versioning scheme for the `apache-airflow-mypy` distributions. +This is to give users confidence about maintaining backwards compatibility in new releases. + +- **Major version** (X.0.0): Breaking changes to plugin interfaces or behavior +- **Minor version** (0.X.0): New features, new plugins, or significant enhancements +- **Patch version** (0.0.X): Bug fixes and minor improvements + +# Prepare Regular apache-airflow-mypy distributions (RC) + +## Generate release notes + +Before releasing, update the `RELEASE_NOTES.rst` file with the changes since the last release. +You can use towncrier to generate release notes from newsfragments: + +```bash +cd dev/mypy +towncrier build --version +``` + +To preview the release notes without writing to the file: + +```bash +towncrier build --version --draft +``` + +Review and edit the generated release notes as needed. + +## Build apache-airflow-mypy distributions for SVN apache upload + +The Release Manager can use the `breeze` tool to build the package: + +```bash +breeze release-management prepare-mypy-distributions \ + --distribution-format both \ + --version-suffix-for-pypi rc1 +``` + +## Build and sign the source and convenience packages + +Follow the same signing process as other Airflow packages: + +```bash +cd dist +for file in *.tar.gz *.whl; do + gpg --armor --detach-sign $file + sha512sum $file > $file.sha512 +done +``` + +## Add tags in git + +Tag the release candidate in git: + +```bash +git tag -s apache-airflow-mypy-rc -m "Apache Airflow Mypy rc" +git push origin apache-airflow-mypy-rc +``` + +## Commit the source packages to Apache SVN repo + +Follow the standard Apache release process for committing to the SVN repository. + +## Publish the distributions to PyPI (release candidates) + +Release candidates should be published to TestPyPI first: + +```bash +twine upload --repository testpypi dist/apache_airflow.mypy-rc* +``` + +## Prepare voting email + +Send a voting email to dev@airflow.apache.org with the following template: + +``` +Subject: [VOTE] Release Apache Airflow Mypy based on rc + +Hello Apache Airflow Community, + +This is a call for the vote to release Apache Airflow Mypy version . + +The release candidate is available at: +https://dist.apache.org/repos/dist/dev/airflow/apache-airflow-mypy-rc/ + +The packages are available at TestPyPI: +https://test.pypi.org/project/apache-airflow-mypy/rc/ + +The vote will be open for at least 72 hours. + +[ ] +1 Approve the release +[ ] +0 No opinion +[ ] -1 Do not release (please provide specific comments) + +Only PMC votes are binding, but everyone is welcome to check and vote. + +Best regards, + +``` + +## Verify the release candidate + +Verify the release by: + +1. Installing from TestPyPI: +```bash +pip install --index-url https://test.pypi.org/simple/ apache-airflow-mypy==rc +``` + +2. Testing the plugins in a mypy configuration: +```ini +[mypy] +plugins = airflow.mypy.plugins.decorators, airflow.mypy.plugins.outputs +``` + +3. Running the test suite if available + +# Publish release + +## Summarize the voting + +Once the vote passes, summarize the results in a reply to the voting thread. + +## Publish release to SVN + +Move the release from dev to release in SVN: + +```bash +svn mv https://dist.apache.org/repos/dist/dev/airflow/apache-airflow-mypy-rc \ + https://dist.apache.org/repos/dist/release/airflow/apache-airflow-mypy- \ + -m "Release Apache Airflow Mypy " +``` + +## Publish the packages to PyPI + +Publish the final release to PyPI: + +```bash +twine upload dist/apache_airflow.mypy--py3-none-any.whl +twine upload dist/apache_airflow.mypy-.tar.gz +``` + +## Add tags in git + +Tag the final release: + +```bash +git tag -s apache-airflow-mypy- -m "Apache Airflow Mypy " +git push origin apache-airflow-mypy- +``` + +## Notify developers of release + +Send an announcement email to dev@airflow.apache.org and announce@apache.org: + +``` +Subject: [ANNOUNCE] Apache Airflow Mypy released + +The Apache Airflow team is pleased to announce the release of Apache Airflow Mypy . + +Apache Airflow Mypy provides Mypy plugins for Apache Airflow to enhance type checking capabilities. + +The release is available at: +https://pypi.org/project/apache-airflow-mypy// + +Release notes: +https://github.com/apache/airflow/blob/main/dev/mypy/RELEASE_NOTES.rst + +Installation: +pip install apache-airflow-mypy + +Usage: +Add to your mypy configuration: +[mypy] +plugins = airflow.mypy.plugins.decorators, airflow.mypy.plugins.outputs + +Cheers, +The Apache Airflow Team +``` diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 926539bd4c361..56f467b5560c0 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -923,6 +923,32 @@ You can use Breeze to generate an airflow-ctl issue when you release new airflow :width: 100% :alt: Breeze release-management generate-issue-content-airflow-ctl +Preparing Apache Airflow Mypy distributions +"""""""""""""""""""""""""""""""""""""""""""" + +You can prepare Apache Airflow Mypy distributions using Breeze: + +.. code-block:: bash + + breeze release-management prepare-mypy-distributions + +This prepares Apache Airflow Mypy .whl package in the dist folder. + +Again, you can specify optional ``--distribution-format`` flag to build selected formats of the Mypy distributions, +default is to build ``both`` type of distributions ``sdist`` and ``wheel``. + +.. code-block:: bash + + breeze release-management prepare-mypy-distributions --distribution-format=wheel + +If you pass ``--tag`` fag, the distribution will create a source tarball release along with sdist. +``--tag`` flag corresponds to actual tag in git. + +.. image:: ./images/output_release-management_prepare-mypy-distributions.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.svg + :width: 100% + :alt: Breeze release-management prepare-mypy-distributions + Publishing the documentation to S3 """""""""""""""""""""""""""""""""" diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index bbbcd76925a3f..bb8f15d386251 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -164,6 +164,9 @@ AIRFLOW_DIST_PATH, AIRFLOW_PROVIDERS_LAST_RELEASE_DATE_PATH, AIRFLOW_ROOT_PATH, + MYPY_DIST_PATH, + MYPY_ROOT_PATH, + MYPY_SOURCES_PATH, OUT_PATH, PROVIDER_METADATA_JSON_PATH, TASK_SDK_DIST_PATH, @@ -296,6 +299,7 @@ class DistributionBuildType(Enum): PROVIDERS = "providers" TASK_SDK = "task-sdk" AIRFLOW_CTL = "airflow-ctl" + MYPY = "mypy" class DistributionPackageInfo(NamedTuple): @@ -759,6 +763,35 @@ def prepare_airflow_ctl_distributions( ) +@release_management_group.command( + name="prepare-mypy-distributions", + help="Prepare sdist/whl distributions of Apache Airflow Mypy.", +) +@option_distribution_format +@option_version_suffix +@option_use_local_hatch +@option_verbose +@option_dry_run +def prepare_mypy_distributions( + distribution_format: str, + version_suffix: str, + use_local_hatch: bool, +): + _prepare_non_core_distributions( + # Argument parameters + distribution_format=distribution_format, + version_suffix=version_suffix, + use_local_hatch=use_local_hatch, + # Distribution specific parameters + root_path=MYPY_ROOT_PATH, + init_file_path=MYPY_SOURCES_PATH / "airflow_mypy" / "__init__.py", + distribution_path=MYPY_DIST_PATH, + distribution_name="mypy", + distribution_pretty_name="Mypy", + full_distribution_pretty_name="Apache Airflow Mypy", + ) + + def provider_action_summary(description: str, message_type: MessageType, packages: list[str]): if packages: console_print(f"{description}: {len(packages)}\n") diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 7f2625a0bcca9..e85f3e085ac90 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -260,6 +260,10 @@ def find_airflow_root_path_to_operate_on() -> Path: AIRFLOW_CTL_SOURCES_PATH = AIRFLOW_CTL_ROOT_PATH / "src" AIRFLOW_CTL_DIST_PATH = AIRFLOW_CTL_ROOT_PATH / "dist" +MYPY_ROOT_PATH = AIRFLOW_ROOT_PATH / "dev" / "mypy" +MYPY_SOURCES_PATH = MYPY_ROOT_PATH / "src" +MYPY_DIST_PATH = MYPY_ROOT_PATH / "dist" + # Same here - do not remove those this is used for past commit retrieval PREVIOUS_AIRFLOW_PROVIDERS_SOURCES_PATH = AIRFLOW_PROVIDERS_ROOT_PATH / "src" PREVIOUS_AIRFLOW_PROVIDERS_NS_PACKAGE_PATH = PREVIOUS_AIRFLOW_PROVIDERS_SOURCES_PATH / "airflow" / "providers" From eaabc44ae02d981b73ccb5182e596eaf8ca6e543 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sun, 22 Mar 2026 19:48:58 +0100 Subject: [PATCH 03/10] fix failed checks --- dev/README_RELEASE_MYPY.md | 16 +++++++++------- .../release_management_commands_config.py | 17 +++++++++++++++++ .../src/airflow_breeze/configure_rich_click.py | 2 ++ dev/mypy/.gitignore | 1 + uv.lock | 13 +++++++++++++ 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/dev/README_RELEASE_MYPY.md b/dev/README_RELEASE_MYPY.md index d51d808889ba6..932eb5a8598a0 100644 --- a/dev/README_RELEASE_MYPY.md +++ b/dev/README_RELEASE_MYPY.md @@ -165,15 +165,17 @@ Best regards, Verify the release by: 1. Installing from TestPyPI: -```bash -pip install --index-url https://test.pypi.org/simple/ apache-airflow-mypy==rc -``` + + ```bash + pip install --index-url https://test.pypi.org/simple/ apache-airflow-mypy==rc + ``` 2. Testing the plugins in a mypy configuration: -```ini -[mypy] -plugins = airflow.mypy.plugins.decorators, airflow.mypy.plugins.outputs -``` + + ```ini + [mypy] + plugins = airflow.mypy.plugins.decorators, airflow.mypy.plugins.outputs + ``` 3. Running the test suite if available diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index 7513b0e033d02..25bb62052412e 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -69,6 +69,13 @@ ], } +RELEASE_MYPY_COMMANDS: dict[str, str | list[str]] = { + "name": "Apache Airflow Mypy release commands", + "commands": [ + "prepare-mypy-distributions", + ], +} + RELEASE_OTHER_COMMANDS: dict[str, str | list[str]] = { "name": "Other release commands", "commands": [ @@ -120,6 +127,16 @@ ], } ], + "breeze release-management prepare-mypy-distributions": [ + { + "name": "Package flags", + "options": [ + "--distribution-format", + "--version-suffix", + "--use-local-hatch", + ], + } + ], "breeze release-management prepare-helm-chart-tarball": [ { "name": "Package flags", diff --git a/dev/breeze/src/airflow_breeze/configure_rich_click.py b/dev/breeze/src/airflow_breeze/configure_rich_click.py index ea2d7c299ff8b..95239d3c6c9a8 100644 --- a/dev/breeze/src/airflow_breeze/configure_rich_click.py +++ b/dev/breeze/src/airflow_breeze/configure_rich_click.py @@ -57,6 +57,7 @@ RELEASE_AIRFLOW_TASK_SDK_COMMANDS, RELEASE_HELM_COMMANDS, RELEASE_MANAGEMENT_PARAMETERS, + RELEASE_MYPY_COMMANDS, RELEASE_OTHER_COMMANDS, RELEASE_PROVIDERS_COMMANDS, ) @@ -144,6 +145,7 @@ RELEASE_PROVIDERS_COMMANDS, RELEASE_AIRFLOW_TASK_SDK_COMMANDS, RELEASE_AIRFLOW_CTL_COMMANDS, + RELEASE_MYPY_COMMANDS, RELEASE_OTHER_COMMANDS, ], "breeze sbom": [SBOM_COMMANDS], diff --git a/dev/mypy/.gitignore b/dev/mypy/.gitignore index 4bee7fc36df78..655276ccf963b 100644 --- a/dev/mypy/.gitignore +++ b/dev/mypy/.gitignore @@ -7,3 +7,4 @@ build/ __pycache__/ *.py[cod] *$py.class +*.iml diff --git a/uv.lock b/uv.lock index 7caa9f691dec3..eefd08472613d 100644 --- a/uv.lock +++ b/uv.lock @@ -161,6 +161,7 @@ members = [ "apache-airflow-e2e-tests", "apache-airflow-helm-tests", "apache-airflow-kubernetes-tests", + "apache-airflow-mypy", "apache-airflow-providers", "apache-airflow-providers-airbyte", "apache-airflow-providers-alibaba", @@ -2049,6 +2050,7 @@ name = "apache-airflow-dev" version = "0.0.1" source = { editable = "dev" } dependencies = [ + { name = "apache-airflow-mypy" }, { name = "beautifulsoup4" }, { name = "click" }, { name = "icalendar" }, @@ -2074,6 +2076,7 @@ mypy = [ [package.metadata] requires-dist = [ + { name = "apache-airflow-mypy", editable = "dev/mypy" }, { name = "beautifulsoup4", specifier = ">=4.12.0" }, { name = "click", specifier = ">=8.3.0" }, { name = "icalendar", specifier = ">=5.0.0" }, @@ -2618,6 +2621,16 @@ requires-dist = [ [package.metadata.requires-dev] mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-mypy" +source = { editable = "dev/mypy" } +dependencies = [ + { name = "mypy" }, +] + +[package.metadata] +requires-dist = [{ name = "mypy", specifier = ">=1.0.0" }] + [[package]] name = "apache-airflow-providers" version = "0.0.1" From 577de81c600edf705bd31144212311597d3dfe58 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sun, 22 Mar 2026 20:48:49 +0100 Subject: [PATCH 04/10] fix failed checks --- .../doc/images/output_release-management.svg | 8 +- .../doc/images/output_release-management.txt | 2 +- ...-management_prepare-mypy-distributions.svg | 128 ++++++++++++++++++ ...-management_prepare-mypy-distributions.txt | 1 + dev/mypy/README.md | 45 +----- 5 files changed, 135 insertions(+), 49 deletions(-) create mode 100644 dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.svg create mode 100644 dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.txt diff --git a/dev/breeze/doc/images/output_release-management.svg b/dev/breeze/doc/images/output_release-management.svg index 2820fd8a0527e..df46a6682d169 100644 --- a/dev/breeze/doc/images/output_release-management.svg +++ b/dev/breeze/doc/images/output_release-management.svg @@ -258,10 +258,10 @@ prepare-task-sdk-distributions            Prepare sdist/whl distributions of Airflow Task SDK.                     ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ airflowctl release commands ────────────────────────────────────────────────────────────────────────────────────────╮ -prepare-airflow-ctl-distributions Prepare sdist/whl distributions of airflowctl.                                   -generate-issue-content-airflow-ctlGenerates content for issue to test airflow-ctl release.                         -generate-airflowctl-changelog     Generates RST changelog for the airflowctl release and prepends it to            -RELEASE_NOTES.rst.                                                               +prepare-airflow-ctl-distributions               Prepare sdist/whl distributions of airflowctl.                     +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Apache Airflow Mypy release commands ───────────────────────────────────────────────────────────────────────────────╮ +prepare-mypy-distributions           Prepare sdist/whl distributions of Apache Airflow Mypy.                       ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ add-back-references     Command to add back references for documentation to make it backward compatible.           diff --git a/dev/breeze/doc/images/output_release-management.txt b/dev/breeze/doc/images/output_release-management.txt index 61eb7f7930c2e..4f4e38c427d80 100644 --- a/dev/breeze/doc/images/output_release-management.txt +++ b/dev/breeze/doc/images/output_release-management.txt @@ -1 +1 @@ -f4c5c17407fe55af6bc2390bef312bf1 +5c767ac05fcd08fd9e6626ee01b14ffe diff --git a/dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.svg new file mode 100644 index 0000000000000..39ebe2ee5eea5 --- /dev/null +++ b/dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.svg @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management prepare-mypy-distributions + + + + + + + + + + +Usage:breeze release-management prepare-mypy-distributions[OPTIONS] + +Prepare sdist/whl distributions of Apache Airflow Mypy. + +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +--distribution-formatFormat of packages. [default: wheel](wheel | sdist | both) +--version-suffix     Version suffix used for PyPI packages (a1, a2, b1, rc1, rc2, .dev0, .dev1, .post1, .post2     +etc.). Note the `.` is need in `.dev0` and `.post`. Might be followed with +local_version     +(TEXT) +--use-local-hatch    Use local hatch instead of docker to build the package. You need to have hatch installed. +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +--verbose-vPrint verbose information about performed steps. +--dry-run-DIf dry-run is set, commands are only printed, not executed. +--help   -hShow this message and exit. +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.txt new file mode 100644 index 0000000000000..5c480ad616e13 --- /dev/null +++ b/dev/breeze/doc/images/output_release-management_prepare-mypy-distributions.txt @@ -0,0 +1 @@ +6a78a6708d57916b90df21bd8a035e8d diff --git a/dev/mypy/README.md b/dev/mypy/README.md index 5f3b7e92b6127..a5b7b86510b94 100644 --- a/dev/mypy/README.md +++ b/dev/mypy/README.md @@ -19,47 +19,4 @@ -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Apache Airflow Mypy Plugins](#apache-airflow-mypy-plugins) - - [Installation](#installation) - - [Usage](#usage) - - [Available Plugins](#available-plugins) - -# Apache Airflow Mypy Plugins - -This package provides Mypy plugins for Apache Airflow to enhance type checking capabilities. - -## Installation - -```bash -pip install apache-airflow-mypy -``` - -## Usage - -Add the plugins to your `mypy.ini` or `pyproject.toml` configuration: - -### Using mypy.ini - -```ini -[mypy] -plugins = airflow_mypy.plugins.decorators, airflow_mypy.plugins.outputs -``` - -### Using pyproject.toml - -```toml -[tool.mypy] -plugins = ["airflow_mypy.plugins.decorators", "airflow_mypy.plugins.outputs"] -``` - -## Available Plugins - -### decorators Plugin - -Provides type checking support for Airflow decorators that modify function signatures. - -### outputs Plugin - -Handles type checking for operator outputs and XComArg types, allowing proper type inference when passing task outputs between tasks. + From 344a7a137b5f8f4bc4d2c4f379c67057538473a9 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sun, 22 Mar 2026 21:04:07 +0100 Subject: [PATCH 05/10] fix failed checks --- dev/mypy/README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/dev/mypy/README.md b/dev/mypy/README.md index a5b7b86510b94..a2ac1baa5ef8a 100644 --- a/dev/mypy/README.md +++ b/dev/mypy/README.md @@ -19,4 +19,48 @@ + +- [Apache Airflow Mypy Plugins](#apache-airflow-mypy-plugins) + - [Installation](#installation) + - [Usage](#usage) + - [Available Plugins](#available-plugins) + + +# Apache Airflow Mypy Plugins + +This package provides Mypy plugins for Apache Airflow to enhance type checking capabilities. + +## Installation + +```bash +pip install apache-airflow-mypy +``` + +## Usage + +Add the plugins to your `mypy.ini` or `pyproject.toml` configuration: + +### Using mypy.ini + +```ini +[mypy] +plugins = airflow_mypy.plugins.decorators, airflow_mypy.plugins.outputs +``` + +### Using pyproject.toml + +```toml +[tool.mypy] +plugins = ["airflow_mypy.plugins.decorators", "airflow_mypy.plugins.outputs"] +``` + +## Available Plugins + +### decorators Plugin + +Provides type checking support for Airflow decorators that modify function signatures. + +### outputs Plugin + +Handles type checking for operator outputs and XComArg types, allowing proper type inference when passing task outputs between tasks. From 7cf946727e364833189d1ef5f9b3e61993de9b4b Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sun, 22 Mar 2026 21:45:53 +0100 Subject: [PATCH 06/10] fix failed checks --- ...utput_setup_check-all-params-in-groups.svg | 80 +++++++++---------- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 60 +++++++------- ...output_setup_regenerate-command-images.txt | 2 +- 4 files changed, 70 insertions(+), 74 deletions(-) diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 93eeb3545d8c7..5b606270774b4 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -1,4 +1,4 @@ - + ci-image:pull | ci-image:save | ci-image:verify | ci:fix-ownership | ci:free-space | ci:get-workflow-info | ci:resource-check | ci:selective-check | ci:set-milestone | ci:upgrade | cleanup | doctor | down | exec | generate-migration-file | issues | issues:unassign | k8s | k8s:build-k8s-image | k8s:configure-cluster  -| k8s:create-cluster | k8s:delete-cluster | k8s:deploy-airflow | k8s:deploy-cluster | k8s:dev | k8s:k9s | -k8s:logs | k8s:run-complete-tests | k8s:setup-env | k8s:shell | k8s:status | k8s:tests |  -k8s:upload-k8s-image | pr | pr:auto-triage | pr:stats | prod-image | prod-image:build | prod-image:load | -prod-image:pull | prod-image:save | prod-image:verify | registry | registry:backfill |  -registry:extract-data | registry:publish-versions | release-management |  -release-management:add-back-references | release-management:check-release-files |  -release-management:clean-old-provider-artifacts | release-management:constraints-version-check |  -release-management:create-minor-branch | release-management:generate-airflowctl-changelog |  -release-management:generate-constraints | release-management:generate-issue-content-airflow-ctl |  -release-management:generate-issue-content-core | release-management:generate-issue-content-helm-chart |  -release-management:generate-issue-content-providers | release-management:generate-providers-metadata |  -release-management:install-provider-distributions | release-management:merge-prod-images |  -release-management:prepare-airflow-ctl-distributions | release-management:prepare-airflow-distributions | -release-management:prepare-helm-chart-package | release-management:prepare-helm-chart-tarball |  -release-management:prepare-provider-distributions | release-management:prepare-provider-documentation |  -release-management:prepare-python-client | release-management:prepare-tarball |  -release-management:prepare-task-sdk-distributions | release-management:publish-docs |  -release-management:publish-docs-to-s3 | release-management:release-prod-images |  -release-management:start-rc-process | release-management:start-release | release-management:tag-providers -| release-management:update-constraints | release-management:update-providers-next-version |  -release-management:verify-provider-distributions | release-management:verify-rc-by-pmc | run | sbom |  -sbom:build-all-airflow-images | sbom:export-dependency-information | sbom:generate-providers-requirements -| sbom:update-sbom-information | setup | setup:autocomplete | setup:check-all-params-in-groups |  -setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts |  -setup:version | shell | start-airflow | testing | testing:airflow-ctl-integration-tests |  -testing:airflow-ctl-tests | testing:airflow-e2e-tests | testing:core-integration-tests |  -testing:core-tests | testing:docker-compose-tests | testing:helm-tests |  -testing:providers-integration-tests | testing:providers-tests | testing:python-api-client-tests |  -testing:system-tests | testing:task-sdk-integration-tests | testing:task-sdk-tests | testing:ui-e2e-tests -| ui | ui:check-translation-completeness | ui:compile-assets | workflow-run | workflow-run:publish-docs) -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ ---verbose-vPrint verbose information about performed steps. ---dry-run-DIf dry-run is set, commands are only printed, not executed. ---help   -hShow this message and exit. -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +| k8s:create-cluster | k8s:delete-cluster | k8s:deploy-airflow | k8s:dev | k8s:k9s | k8s:logs |  +k8s:run-complete-tests | k8s:setup-env | k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | pr | +pr:auto-triage | prod-image | prod-image:build | prod-image:load | prod-image:pull | prod-image:save |  +prod-image:verify | registry | registry:backfill | registry:extract-data | registry:publish-versions |  +release-management | release-management:add-back-references | release-management:check-release-files |  +release-management:clean-old-provider-artifacts | release-management:constraints-version-check |  +release-management:create-minor-branch | release-management:generate-constraints |  +release-management:generate-issue-content-core | release-management:generate-issue-content-helm-chart |  +release-management:generate-issue-content-providers | release-management:generate-providers-metadata |  +release-management:install-provider-distributions | release-management:merge-prod-images |  +release-management:prepare-airflow-ctl-distributions | release-management:prepare-airflow-distributions | +release-management:prepare-helm-chart-package | release-management:prepare-helm-chart-tarball |  +release-management:prepare-mypy-distributions | release-management:prepare-provider-distributions |  +release-management:prepare-provider-documentation | release-management:prepare-python-client |  +release-management:prepare-tarball | release-management:prepare-task-sdk-distributions |  +release-management:publish-docs | release-management:publish-docs-to-s3 |  +release-management:release-prod-images | release-management:start-rc-process |  +release-management:start-release | release-management:tag-providers |  +release-management:update-constraints | release-management:update-providers-next-version |  +release-management:verify-provider-distributions | release-management:verify-rc-by-pmc | run | sbom |  +sbom:build-all-airflow-images | sbom:export-dependency-information | sbom:generate-providers-requirements +| sbom:update-sbom-information | setup | setup:autocomplete | setup:check-all-params-in-groups |  +setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts |  +setup:version | shell | start-airflow | testing | testing:airflow-ctl-integration-tests |  +testing:airflow-ctl-tests | testing:airflow-e2e-tests | testing:core-integration-tests |  +testing:core-tests | testing:docker-compose-tests | testing:helm-tests |  +testing:providers-integration-tests | testing:providers-tests | testing:python-api-client-tests |  +testing:system-tests | testing:task-sdk-integration-tests | testing:task-sdk-tests | testing:ui-e2e-tests +| ui | ui:check-translation-completeness | ui:compile-assets | workflow-run | workflow-run:publish-docs) +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +--verbose-vPrint verbose information about performed steps. +--dry-run-DIf dry-run is set, commands are only printed, not executed. +--help   -hShow this message and exit. +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt index 1956792e47d3e..b5ce7e19e2d4d 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt @@ -1 +1 @@ -6a605c405b9bd7f51d3ba9b294de9231 +dae2ff77eea9c05c5577b1df97fd7d0a diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg index ddfe50c681981..58070708cbe98 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg @@ -223,37 +223,37 @@ ci:get-workflow-info | ci:resource-check | ci:selective-check | ci:set-milestone | ci:upgrade |  cleanup | doctor | down | exec | generate-migration-file | issues | issues:unassign | k8s |  k8s:build-k8s-image | k8s:configure-cluster | k8s:create-cluster | k8s:delete-cluster |  -k8s:deploy-airflow | k8s:deploy-cluster | k8s:dev | k8s:k9s | k8s:logs | k8s:run-complete-tests |  -k8s:setup-env | k8s:shell | k8s:status | k8s:tests | k8s:upload-k8s-image | pr | pr:auto-triage |  -pr:stats | prod-image | prod-image:build | prod-image:load | prod-image:pull | prod-image:save |  -prod-image:verify | registry | registry:backfill | registry:extract-data | registry:publish-versions | -release-management | release-management:add-back-references | release-management:check-release-files | +k8s:deploy-airflow | k8s:dev | k8s:k9s | k8s:logs | k8s:run-complete-tests | k8s:setup-env | k8s:shell +| k8s:status | k8s:tests | k8s:upload-k8s-image | pr | pr:auto-triage | prod-image | prod-image:build  +| prod-image:load | prod-image:pull | prod-image:save | prod-image:verify | registry |  +registry:backfill | registry:extract-data | registry:publish-versions | release-management |  +release-management:add-back-references | release-management:check-release-files |  release-management:clean-old-provider-artifacts | release-management:constraints-version-check |  -release-management:create-minor-branch | release-management:generate-airflowctl-changelog |  -release-management:generate-constraints | release-management:generate-issue-content-airflow-ctl |  -release-management:generate-issue-content-core | release-management:generate-issue-content-helm-chart  -| release-management:generate-issue-content-providers | release-management:generate-providers-metadata -| release-management:install-provider-distributions | release-management:merge-prod-images |  -release-management:prepare-airflow-ctl-distributions |  -release-management:prepare-airflow-distributions | release-management:prepare-helm-chart-package |  -release-management:prepare-helm-chart-tarball | release-management:prepare-provider-distributions |  -release-management:prepare-provider-documentation | release-management:prepare-python-client |  -release-management:prepare-tarball | release-management:prepare-task-sdk-distributions |  -release-management:publish-docs | release-management:publish-docs-to-s3 |  -release-management:release-prod-images | release-management:start-rc-process |  -release-management:start-release | release-management:tag-providers |  -release-management:update-constraints | release-management:update-providers-next-version |  -release-management:verify-provider-distributions | release-management:verify-rc-by-pmc | run | sbom |  -sbom:build-all-airflow-images | sbom:export-dependency-information |  -sbom:generate-providers-requirements | sbom:update-sbom-information | setup | setup:autocomplete |  -setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images | setup:self-upgrade -| setup:synchronize-local-mounts | setup:version | shell | start-airflow | testing |  -testing:airflow-ctl-integration-tests | testing:airflow-ctl-tests | testing:airflow-e2e-tests |  -testing:core-integration-tests | testing:core-tests | testing:docker-compose-tests |  -testing:helm-tests | testing:providers-integration-tests | testing:providers-tests |  -testing:python-api-client-tests | testing:system-tests | testing:task-sdk-integration-tests |  -testing:task-sdk-tests | testing:ui-e2e-tests | ui | ui:check-translation-completeness |  -ui:compile-assets | workflow-run | workflow-run:publish-docs) +release-management:create-minor-branch | release-management:generate-constraints |  +release-management:generate-issue-content-core | release-management:generate-issue-content-helm-chart  +| release-management:generate-issue-content-providers | release-management:generate-providers-metadata +| release-management:install-provider-distributions | release-management:merge-prod-images |  +release-management:prepare-airflow-ctl-distributions |  +release-management:prepare-airflow-distributions | release-management:prepare-helm-chart-package |  +release-management:prepare-helm-chart-tarball | release-management:prepare-mypy-distributions |  +release-management:prepare-provider-distributions | release-management:prepare-provider-documentation  +| release-management:prepare-python-client | release-management:prepare-tarball |  +release-management:prepare-task-sdk-distributions | release-management:publish-docs |  +release-management:publish-docs-to-s3 | release-management:release-prod-images |  +release-management:start-rc-process | release-management:start-release |  +release-management:tag-providers | release-management:update-constraints |  +release-management:update-providers-next-version | release-management:verify-provider-distributions |  +release-management:verify-rc-by-pmc | run | sbom | sbom:build-all-airflow-images |  +sbom:export-dependency-information | sbom:generate-providers-requirements |  +sbom:update-sbom-information | setup | setup:autocomplete | setup:check-all-params-in-groups |  +setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts | +setup:version | shell | start-airflow | testing | testing:airflow-ctl-integration-tests |  +testing:airflow-ctl-tests | testing:airflow-e2e-tests | testing:core-integration-tests |  +testing:core-tests | testing:docker-compose-tests | testing:helm-tests |  +testing:providers-integration-tests | testing:providers-tests | testing:python-api-client-tests |  +testing:system-tests | testing:task-sdk-integration-tests | testing:task-sdk-tests |  +testing:ui-e2e-tests | ui | ui:check-translation-completeness | ui:compile-assets | workflow-run |  +workflow-run:publish-docs) --check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used   together with --command flag or --force.                                                               ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt index 8538e42bc7ae6..cf22549a326d2 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt @@ -1 +1 @@ -842d090a8c3d4ea38e9eeedf15d1341e +307baf61691900bafe8131f97ba35579 From 5c0cb45e7afb4f80568c0a5ceba24cd4a9a68dba Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Sun, 12 Apr 2026 21:38:14 +0200 Subject: [PATCH 07/10] Add hussein-awala as CODEOWNER for dev/mypy/ Made-with: Cursor --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2da9dbc9c27c9..03701a6e7de81 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -121,6 +121,7 @@ airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/ @Lee-W @jason810496 @guan # Dev tools /.github/workflows/ @potiuk @ashb @gopidesupavan @amoghrajesh @jscheffl @bugraoz93 @kaxil @jason810496 /dev/ @potiuk @ashb @gopidesupavan @amoghrajesh @jscheffl @bugraoz93 @jason810496 @jedcunningham @ephraimbuddy @choo121600 @vatsrahul1001 +/dev/mypy/ @hussein-awala /dev/react-plugin-tools/ @pierrejeambrun @bbovenzi /docker-tests/ @potiuk @ashb @gopidesupavan @jason810496 /kubernetes-tests/ @potiuk @ashb @gopidesupavan @jason810496 From 7d63a69296701cbb0dff850fe36bf5bb506a9f9a Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Tue, 21 Apr 2026 23:14:13 +0200 Subject: [PATCH 08/10] Rebase onto main and regenerate auto-generated files Made-with: Cursor --- .../doc/images/output_release-management.svg | 54 +++++++----- .../doc/images/output_release-management.txt | 2 +- ...utput_setup_check-all-params-in-groups.svg | 84 ++++++++++-------- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 86 ++++++++++--------- ...output_setup_regenerate-command-images.txt | 2 +- pyproject.toml | 2 + uv.lock | 11 ++- 8 files changed, 137 insertions(+), 106 deletions(-) diff --git a/dev/breeze/doc/images/output_release-management.svg b/dev/breeze/doc/images/output_release-management.svg index df46a6682d169..3b6e9dff155d5 100644 --- a/dev/breeze/doc/images/output_release-management.svg +++ b/dev/breeze/doc/images/output_release-management.svg @@ -1,4 +1,4 @@ - + prepare-task-sdk-distributions            Prepare sdist/whl distributions of Airflow Task SDK.                     ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ airflowctl release commands ────────────────────────────────────────────────────────────────────────────────────────╮ -prepare-airflow-ctl-distributions               Prepare sdist/whl distributions of airflowctl.                     -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Apache Airflow Mypy release commands ───────────────────────────────────────────────────────────────────────────────╮ -prepare-mypy-distributions           Prepare sdist/whl distributions of Apache Airflow Mypy.                       +prepare-airflow-ctl-distributions Prepare sdist/whl distributions of airflowctl.                                   +generate-issue-content-airflow-ctlGenerates content for issue to test airflow-ctl release.                         +generate-airflowctl-changelog     Generates RST changelog for the airflowctl release and prepends it to            +RELEASE_NOTES.rst.                                                               ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ -add-back-references     Command to add back references for documentation to make it backward compatible.           -prepare-python-client   Prepares python client packages.                                                           -publish-docs            Command to publish generated documentation to airflow-site                                 -generate-constraints    Generates pinned constraint files with all extras from pyproject.toml in parallel.         -update-constraints      Update released constraints with manual changes.                                           -publish-docs-to-s3      Publishes docs to S3.                                                                      -verify-rc-by-pmc        [EXPERIMENTAL] Verify a release candidate for PMC voting.                                  -check-release-files     Verify that all expected packages are present in Apache Airflow svn.                       -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -constraints-version-checkCheck constraints against released versions of packages.                                  -merge-prod-images        Merge production images in DockerHub based on digest files (needs DockerHub permissions). -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Apache Airflow Mypy release commands ───────────────────────────────────────────────────────────────────────────────╮ +prepare-mypy-distributions           Prepare sdist/whl distributions of Apache Airflow Mypy.                       +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ +add-back-references     Command to add back references for documentation to make it backward compatible.           +prepare-python-client   Prepares python client packages.                                                           +publish-docs            Command to publish generated documentation to airflow-site                                 +generate-constraints    Generates pinned constraint files with all extras from pyproject.toml in parallel.         +update-constraints      Update released constraints with manual changes.                                           +publish-docs-to-s3      Publishes docs to S3.                                                                      +verify-rc-by-pmc        [EXPERIMENTAL] Verify a release candidate for PMC voting.                                  +check-release-files     Verify that all expected packages are present in Apache Airflow svn.                       +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +constraints-version-checkCheck constraints against released versions of packages.                                  +merge-prod-images        Merge production images in DockerHub based on digest files (needs DockerHub permissions). +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management.txt b/dev/breeze/doc/images/output_release-management.txt index 4f4e38c427d80..83fb236b58d40 100644 --- a/dev/breeze/doc/images/output_release-management.txt +++ b/dev/breeze/doc/images/output_release-management.txt @@ -1 +1 @@ -5c767ac05fcd08fd9e6626ee01b14ffe +556d85db7551620690a42f59632bb27c diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 5b606270774b4..4606f17493201 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -1,4 +1,4 @@ - +