From 70867b2b7ce216349ba3d4255c3bed4509fc9026 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 7 Jul 2026 22:32:27 +0100 Subject: [PATCH 01/19] Add data quality provider --- .github/boring-cyborg.yml | 3 + providers/dq/.gitignore | 3 + providers/dq/.pre-commit-config.yaml | 36 ++ providers/dq/LICENSE | 201 +++++++ providers/dq/NOTICE | 5 + providers/dq/README.rst | 52 ++ providers/dq/docs/agents.rst | 55 ++ providers/dq/docs/assets.rst | 66 +++ providers/dq/docs/changelog.rst | 33 ++ providers/dq/docs/commits.rst | 35 ++ providers/dq/docs/conf.py | 27 + providers/dq/docs/configurations-ref.rst | 19 + providers/dq/docs/decorators.rst | 52 ++ providers/dq/docs/index.rst | 175 ++++++ .../installing-providers-from-sources.rst | 18 + providers/dq/docs/operators.rst | 119 ++++ providers/dq/docs/rules.rst | 221 +++++++ providers/dq/docs/security.rst | 18 + providers/dq/provider.yaml | 77 +++ providers/dq/pyproject.toml | 152 +++++ providers/dq/src/airflow/__init__.py | 17 + .../dq/src/airflow/providers/__init__.py | 17 + .../dq/src/airflow/providers/dq/__init__.py | 37 ++ .../dq/src/airflow/providers/dq/assets.py | 172 ++++++ .../airflow/providers/dq/backends/__init__.py | 32 ++ .../providers/dq/backends/object_storage.py | 259 +++++++++ .../providers/dq/decorators/__init__.py | 17 + .../providers/dq/decorators/dq_check.py | 108 ++++ .../airflow/providers/dq/engines/__init__.py | 21 + .../src/airflow/providers/dq/engines/sql.py | 137 +++++ .../providers/dq/example_dags/__init__.py | 16 + .../example_dq_check_custom_sql.py | 112 ++++ .../example_dq_check_decorator_dynamic.py | 94 +++ .../example_dq_llm_generated_ruleset.py | 128 +++++ .../example_dq_require_quality.py | 120 ++++ .../example_dq_ruleset_from_yaml.py | 83 +++ .../dq/example_dags/orders_ruleset.yaml | 36 ++ .../dq/src/airflow/providers/dq/exceptions.py | 25 + .../airflow/providers/dq/get_provider_info.py | 67 +++ .../providers/dq/operators/__init__.py | 21 + .../providers/dq/operators/dq_check.py | 247 ++++++++ .../dq/src/airflow/providers/dq/results.py | 116 ++++ .../airflow/providers/dq/rules/__init__.py | 39 ++ .../src/airflow/providers/dq/rules/checks.py | 82 +++ .../dq/src/airflow/providers/dq/rules/rule.py | 279 +++++++++ .../dq/skills/dq-rule-authoring/SKILL.md | 174 ++++++ .../references/ruleset.schema.json | 241 ++++++++ .../airflow/providers/dq/version_compat.py | 37 ++ providers/dq/tests/conftest.py | 19 + providers/dq/tests/system/__init__.py | 17 + providers/dq/tests/system/dq/__init__.py | 16 + .../dq/tests/system/dq/example_dq_check.py | 354 ++++++++++++ providers/dq/tests/unit/__init__.py | 17 + providers/dq/tests/unit/dq/__init__.py | 16 + .../dq/tests/unit/dq/backends/__init__.py | 16 + .../unit/dq/backends/test_object_storage.py | 357 ++++++++++++ .../dq/tests/unit/dq/decorators/__init__.py | 16 + .../tests/unit/dq/decorators/test_dq_check.py | 123 ++++ .../dq/tests/unit/dq/engines/__init__.py | 16 + .../dq/tests/unit/dq/engines/test_sql.py | 118 ++++ .../dq/tests/unit/dq/operators/__init__.py | 16 + .../tests/unit/dq/operators/test_dq_check.py | 247 ++++++++ providers/dq/tests/unit/dq/rules/__init__.py | 16 + .../dq/tests/unit/dq/rules/test_checks.py | 30 + providers/dq/tests/unit/dq/rules/test_rule.py | 264 +++++++++ providers/dq/tests/unit/dq/test_assets.py | 170 ++++++ providers/dq/tests/unit/dq/test_results.py | 82 +++ pyproject.toml | 10 + scripts/ci/docker-compose/remove-sources.yml | 1 + scripts/ci/docker-compose/tests-sources.yml | 1 + scripts/ci/prek/generate_dq_ruleset_schema.py | 98 ++++ uv.lock | 544 ++++++++++-------- 72 files changed, 6422 insertions(+), 243 deletions(-) create mode 100644 providers/dq/.gitignore create mode 100644 providers/dq/.pre-commit-config.yaml create mode 100644 providers/dq/LICENSE create mode 100644 providers/dq/NOTICE create mode 100644 providers/dq/README.rst create mode 100644 providers/dq/docs/agents.rst create mode 100644 providers/dq/docs/assets.rst create mode 100644 providers/dq/docs/changelog.rst create mode 100644 providers/dq/docs/commits.rst create mode 100644 providers/dq/docs/conf.py create mode 100644 providers/dq/docs/configurations-ref.rst create mode 100644 providers/dq/docs/decorators.rst create mode 100644 providers/dq/docs/index.rst create mode 100644 providers/dq/docs/installing-providers-from-sources.rst create mode 100644 providers/dq/docs/operators.rst create mode 100644 providers/dq/docs/rules.rst create mode 100644 providers/dq/docs/security.rst create mode 100644 providers/dq/provider.yaml create mode 100644 providers/dq/pyproject.toml create mode 100644 providers/dq/src/airflow/__init__.py create mode 100644 providers/dq/src/airflow/providers/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/assets.py create mode 100644 providers/dq/src/airflow/providers/dq/backends/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/backends/object_storage.py create mode 100644 providers/dq/src/airflow/providers/dq/decorators/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/decorators/dq_check.py create mode 100644 providers/dq/src/airflow/providers/dq/engines/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/engines/sql.py create mode 100644 providers/dq/src/airflow/providers/dq/example_dags/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py create mode 100644 providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py create mode 100644 providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py create mode 100644 providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py create mode 100644 providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py create mode 100644 providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml create mode 100644 providers/dq/src/airflow/providers/dq/exceptions.py create mode 100644 providers/dq/src/airflow/providers/dq/get_provider_info.py create mode 100644 providers/dq/src/airflow/providers/dq/operators/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/operators/dq_check.py create mode 100644 providers/dq/src/airflow/providers/dq/results.py create mode 100644 providers/dq/src/airflow/providers/dq/rules/__init__.py create mode 100644 providers/dq/src/airflow/providers/dq/rules/checks.py create mode 100644 providers/dq/src/airflow/providers/dq/rules/rule.py create mode 100644 providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md create mode 100644 providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json create mode 100644 providers/dq/src/airflow/providers/dq/version_compat.py create mode 100644 providers/dq/tests/conftest.py create mode 100644 providers/dq/tests/system/__init__.py create mode 100644 providers/dq/tests/system/dq/__init__.py create mode 100644 providers/dq/tests/system/dq/example_dq_check.py create mode 100644 providers/dq/tests/unit/__init__.py create mode 100644 providers/dq/tests/unit/dq/__init__.py create mode 100644 providers/dq/tests/unit/dq/backends/__init__.py create mode 100644 providers/dq/tests/unit/dq/backends/test_object_storage.py create mode 100644 providers/dq/tests/unit/dq/decorators/__init__.py create mode 100644 providers/dq/tests/unit/dq/decorators/test_dq_check.py create mode 100644 providers/dq/tests/unit/dq/engines/__init__.py create mode 100644 providers/dq/tests/unit/dq/engines/test_sql.py create mode 100644 providers/dq/tests/unit/dq/operators/__init__.py create mode 100644 providers/dq/tests/unit/dq/operators/test_dq_check.py create mode 100644 providers/dq/tests/unit/dq/rules/__init__.py create mode 100644 providers/dq/tests/unit/dq/rules/test_checks.py create mode 100644 providers/dq/tests/unit/dq/rules/test_rule.py create mode 100644 providers/dq/tests/unit/dq/test_assets.py create mode 100644 providers/dq/tests/unit/dq/test_results.py create mode 100755 scripts/ci/prek/generate_dq_ruleset_schema.py diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index cd31dc07f4036..d874d08dfee7f 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -144,6 +144,9 @@ labelPRBasedOnFilePath: provider:docker: - providers/docker/** + provider:dq: + - providers/dq/** + provider:edge: - providers/edge3/** diff --git a/providers/dq/.gitignore b/providers/dq/.gitignore new file mode 100644 index 0000000000000..f454b41e73c0a --- /dev/null +++ b/providers/dq/.gitignore @@ -0,0 +1,3 @@ +# Do not commit hash files, this is just to speed-up local builds +www-hash.txt +*.iml diff --git a/providers/dq/.pre-commit-config.yaml b/providers/dq/.pre-commit-config.yaml new file mode 100644 index 0000000000000..8aedb6660a14a --- /dev/null +++ b/providers/dq/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 + node: 22.19.0 + golang: 1.24.0 +repos: + - repo: local + hooks: + - id: generate-dq-ruleset-schema + name: Generate Data Quality RuleSet schema + language: python + entry: ../../scripts/ci/prek/generate_dq_ruleset_schema.py + pass_filenames: false + always_run: true + files: > + (?x) + ^src/airflow/providers/dq/rules/.*\.py$| + ^src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset\.schema\.json$ diff --git a/providers/dq/LICENSE b/providers/dq/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/dq/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/dq/NOTICE b/providers/dq/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/dq/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/dq/README.rst b/providers/dq/README.rst new file mode 100644 index 0000000000000..a6f6ba496bdbc --- /dev/null +++ b/providers/dq/README.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Package ``apache-airflow-providers-dq`` + +Release: ``0.1.0`` + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +Provider package +---------------- + +This package is for the ``dq`` provider. +All classes for this package are included in the ``airflow.providers.dq`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-dq``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== diff --git a/providers/dq/docs/agents.rst b/providers/dq/docs/agents.rst new file mode 100644 index 0000000000000..cc820cf88b416 --- /dev/null +++ b/providers/dq/docs/agents.rst @@ -0,0 +1,55 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:agents: + +Generating rules with an LLM +============================== + +Writing a :class:`~airflow.providers.dq.rules.RuleSet` by hand for every table doesn't scale. +An LLM can propose one from a table's column definitions instead, given the check catalog as +context. + +The ``dq-rule-authoring`` skill +--------------------------------- + +This provider ships an `Agent Skill `__ at +``airflow/providers/dq/skills/dq-rule-authoring/``: a ``SKILL.md`` documenting the +``RuleSet``/``DQRule`` fields and check catalog, plus a generated JSON Schema +(``references/ruleset.schema.json``) for validation. + +Point ``common.ai``'s :doc:`AgentSkillsToolset ` at +it, and give the model ``output_type=RuleSet`` so pydantic-ai validates -- and self-corrects -- +its output before the task completes: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_task_dq_generate_ruleset_with_llm] + :end-before: [END howto_task_dq_generate_ruleset_with_llm] + +Requires ``apache-airflow-providers-common-ai[skills]`` and a configured ``llm_conn_id``. + +Wiring the result into a check +--------------------------------- + +``@task.dq_check`` can leave ``ruleset=`` unset and return the LLM task's result at execution +time instead: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_decorator_dq_check_llm_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_llm_runtime_ruleset] diff --git a/providers/dq/docs/assets.rst b/providers/dq/docs/assets.rst new file mode 100644 index 0000000000000..538952f50a085 --- /dev/null +++ b/providers/dq/docs/assets.rst @@ -0,0 +1,66 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:assets: + +Assets and quality gating +============================ + +Quality rules can travel with the :class:`~airflow.sdk.Asset` they describe instead of being +scattered across every Dag that checks it, and a downstream consumer Dag can refuse to run when +the data it was triggered by did not meet a minimum quality bar. + +Attaching a ruleset to an asset +---------------------------------- + +:func:`~airflow.providers.dq.assets.asset_quality` stores ruleset, connection, and table +configuration inside ``Asset.extra`` under the ``airflow.dq`` key, so it is serialized with the +Dag and needs no Airflow core changes: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_asset_quality] + :end-before: [END howto_asset_quality] + +Pass the resulting asset to :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` +via ``asset=`` (see :doc:`operators`) instead of ``table``/``ruleset``/``conn_id``: the operator +resolves all three from the asset's config, adds the asset to its own outlets, and attaches its +summary -- including the quality ``score`` used below -- to the asset event. + +Gating a consumer Dag on quality +------------------------------------ + +:func:`~airflow.providers.dq.assets.require_quality` builds a ``@task.short_circuit`` task that +reads the ``score`` off the asset event that triggered the current run, and skips every +downstream task when that event has no quality summary at all, or its score is below +``min_score``: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_require_quality] + :end-before: [END howto_require_quality] + +Put the gate first in a Dag scheduled by the asset, and chain everything else after it: + +.. code-block:: python + + with DAG("orders_consumer", schedule=orders_asset) as consumer: + gate = require_quality(orders_asset, min_score=0.95) + gate >> process_orders() + +``min_score`` must be between ``0`` and ``1``; the check considers only the *most recent* +triggering event for the asset when a run was triggered by several. diff --git a/providers/dq/docs/changelog.rst b/providers/dq/docs/changelog.rst new file mode 100644 index 0000000000000..3441e711a8345 --- /dev/null +++ b/providers/dq/docs/changelog.rst @@ -0,0 +1,33 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there + are some breaking changes and you want to add an explanation to the users on how they + are supposed to deal with them. The changelog is updated and maintained semi-automatically + by release manager. + +``apache-airflow-providers-dq`` + + +Changelog +--------- + +0.1.0 +..... + +Initial version of the provider. diff --git a/providers/dq/docs/commits.rst b/providers/dq/docs/commits.rst new file mode 100644 index 0000000000000..6f999c6f0f339 --- /dev/null +++ b/providers/dq/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-dq +------------------------------------------------------ + +``Data Quality Provider`` + + +This is detailed commit list of changes for versions provider package: ``dq``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/dq/docs/conf.py b/providers/dq/docs/conf.py new file mode 100644 index 0000000000000..f133bf86729a7 --- /dev/null +++ b/providers/dq/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-dq" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/dq/docs/configurations-ref.rst b/providers/dq/docs/configurations-ref.rst new file mode 100644 index 0000000000000..a52b21b2e5679 --- /dev/null +++ b/providers/dq/docs/configurations-ref.rst @@ -0,0 +1,19 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../devel-common/src/sphinx_exts/includes/providers-configurations-ref.rst +.. include:: /../../../devel-common/src/sphinx_exts/includes/sections-and-options.rst diff --git a/providers/dq/docs/decorators.rst b/providers/dq/docs/decorators.rst new file mode 100644 index 0000000000000..257e0de8bb697 --- /dev/null +++ b/providers/dq/docs/decorators.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/decorator:dq_check: + +``@task.dq_check`` +===================== + +``@task.dq_check`` wraps :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` in +the TaskFlow API. ``ruleset`` may be declared as a decorator argument when it exists at +Dag-parse time, or returned by the decorated function as a runtime ruleset. ``table`` or +``asset`` are declared as decorator arguments exactly like the plain operator. The decorated +function is optional plumbing on top: return ``None`` to run the check exactly as declared. + +.. exampleinclude:: /../tests/system/dq/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_decorator_dq_check] + :end-before: [END howto_decorator_dq_check] + +Runtime rule sets +-------------------- + +Return a :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a YAML path to use a +ruleset that is only known at task-execution time -- for example one produced by an upstream +task, loaded from a Variable, or generated by an LLM. Return ``None`` to use the ruleset declared +on the decorator. + +Swapping in a different ruleset at execution time: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py + :language: python + :start-after: [START howto_decorator_dq_check_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_runtime_ruleset] + +Everything documented for the plain operator in :doc:`operators` -- ``fail_on``, persistence, +``asset``, ``partition_clause``, ``custom_sql`` -- applies unchanged; the decorator only adds the +optional runtime ruleset step before the check runs. diff --git a/providers/dq/docs/index.rst b/providers/dq/docs/index.rst new file mode 100644 index 0000000000000..0559b586e2f16 --- /dev/null +++ b/providers/dq/docs/index.rst @@ -0,0 +1,175 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-dq`` +=============================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Rules, rule sets, and checks + Operators + Decorators + Assets and quality gating + Generating rules with an LLM + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Configuration + Python API <_api/airflow/providers/dq/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Installing from sources + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/dq/index> + + +apache-airflow-providers-dq package +----------------------------------- + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +See :doc:`rules` for the built-in check catalog (and its cross-database caveats), +:doc:`operators`/:doc:`decorators` for running checks, and :doc:`assets` for attaching rules to +an asset and gating a downstream Dag on its quality score. + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``dq`` provider. +All classes for this package are included in the ``airflow.providers.dq`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-dq``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-dq package +------------------------------------------------------ + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. +Checks run through ``common.sql`` DB-API hooks; results are persisted to a +configurable results store (object storage or local files) so task, run, +and rule-level quality can be inspected over time. + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``dq`` provider. +All classes for this package are included in the ``airflow.providers.dq`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-dq``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-dq 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dq 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/dq/docs/installing-providers-from-sources.rst b/providers/dq/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..a72b45ffaa6e8 --- /dev/null +++ b/providers/dq/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/dq/docs/operators.rst b/providers/dq/docs/operators.rst new file mode 100644 index 0000000000000..70243471e455a --- /dev/null +++ b/providers/dq/docs/operators.rst @@ -0,0 +1,119 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/operator:DQCheckOperator: + +``DQCheckOperator`` +===================== + +Use :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` to run a +:doc:`ruleset ` against a table and persist per-rule results to the configured results +store. Every rule is evaluated and recorded regardless of the task outcome -- a rule failing +doesn't stop the others from running, and the full set of results is always written before the +task decides whether to fail. + +Basic usage +------------ + +Pass a connection, table, and ruleset directly: + +.. exampleinclude:: /../tests/system/dq/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dq_check] + :end-before: [END howto_operator_dq_check] + +Parameters +----------- + +- ``ruleset`` -- a :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path to a + YAML ruleset file (see :doc:`rules`). Optional when ``asset`` carries one. +- ``table`` -- the table to check. Optional when ``asset`` carries one (falling back to the + asset's name). +- ``asset`` -- an :class:`~airflow.sdk.Asset` decorated with + :func:`~airflow.providers.dq.assets.asset_quality`. Supplies defaults for ``ruleset``, + ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's + outlets so its asset events carry the check summary -- see :doc:`assets`. +- ``partition_clause`` -- predicate ANDed into every built-in check's ``WHERE`` clause, e.g. + ``"ds = '{{ ds }}'"`` (templated). +- ``fail_on`` -- severity that fails the task: + + - ``error`` (default) -- only ``error``-severity rule failures fail the task; ``warn`` + failures are recorded but don't fail it. + - ``warn`` -- any rule failure (``error`` or ``warn`` severity) fails the task. + - ``never`` -- rule failures never fail the task; only an execution error (the check query + itself failing) does. + +- ``conn_id`` -- connection to the database to check, any ``common.sql`` ``DbApiHook``- + compatible type. +- ``database`` -- optional database/schema, overriding the connection's default. + +Regardless of ``fail_on``, an execution error -- the check query itself failing, as opposed to +a rule failing its condition -- always fails the task; there's no way to check data whose query +can't even run. + +Persisting results +-------------------- + +Results are persisted to the backend configured under ``[dq] results_path`` (see +:doc:`configurations-ref`). When that's unset, checks still run and the task still passes or +fails normally -- only persisted data quality history is unavailable. There is no +per-operator override: every check in a deployment shares one results store, so history stays +available across tasks and Dags without stitching together several stores. + +Checking an asset +-------------------- + +Attach a ruleset to an :class:`~airflow.sdk.Asset` with +:func:`~airflow.providers.dq.assets.asset_quality`, then pass the asset instead of ``table``/ +``ruleset``/``conn_id``: + +.. exampleinclude:: /../tests/system/dq/example_dq_check.py + :language: python + :start-after: [START howto_operator_dq_check_asset] + :end-before: [END howto_operator_dq_check_asset] + +The operator adds the asset to its own outlets automatically, and attaches the check's summary +(including its quality ``score``) to the asset event -- which is what makes +:func:`~airflow.providers.dq.assets.require_quality` (see :doc:`assets`) able to gate a +downstream consumer Dag on it. + +``custom_sql`` checks +------------------------ + +Built-in checks are all single-column. For cross-column comparisons, joins, or anything the +catalog doesn't cover, use ``custom_sql``: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +See :doc:`rules` for the full built-in check catalog and the ``custom_sql`` grammar. + +Loading a ruleset from YAML +------------------------------ + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +TaskFlow decorator +-------------------- + +See :doc:`decorators` for the ``@task.dq_check`` equivalent, including runtime rule sets. diff --git a/providers/dq/docs/rules.rst b/providers/dq/docs/rules.rst new file mode 100644 index 0000000000000..025ecd4061a24 --- /dev/null +++ b/providers/dq/docs/rules.rst @@ -0,0 +1,221 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:rules: + +Rules, rule sets, and checks +============================ + +Rules are data, not code. A :class:`~airflow.providers.dq.rules.RuleSet` is a named tuple of +:class:`~airflow.providers.dq.rules.DQRule` -- plain, serializable objects with no behavior of +their own. They describe *what* to check; :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` +(see :doc:`operators`) is what actually runs them. Rules are usually written by hand, but they +can also be proposed by an LLM -- see :doc:`agents`. + +.. code-block:: python + + from airflow.providers.dq.rules import DQRule, RuleSet + + orders_ruleset = RuleSet( + name="orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="row_count_present", + check="row_count", + condition={"greater_than": 0}, + ), + ), + ) + +``DQRule`` fields +------------------ + +- ``name`` -- unique within its ruleset. Shows up in data quality results and logs. +- ``check`` -- one of the built-in checks below, or ``custom_sql``. +- ``condition`` -- the pass/fail condition for the observed value; see `Conditions`_. +- ``column`` -- target column. Required for column-level built-in checks, unused for + ``row_count`` and ``custom_sql``. +- ``sql`` -- a SQL statement returning a single scalar. Required for, and only valid with, + ``check="custom_sql"``. May reference the table being checked as ``{table}``. +- ``severity`` -- ``error`` (default) or ``warn``. Controls whether a failing rule fails the + task, subject to the operator's ``fail_on`` (see :doc:`operators`). +- ``partition_clause`` -- extra SQL predicate ANDed into this rule's ``WHERE`` clause, e.g. + ``"region = 'EU'"``. Combines with the operator-level ``partition_clause``, if any. +- ``previous_name`` -- set when renaming a rule so its execution history stays continuous + (see `Identity and history`_). +- ``description`` -- optional human-readable text shown in data quality results. + When omitted, the provider generates a short default description from the rule and condition. +- ``dimension`` -- one of ``completeness``, ``uniqueness``, ``validity``, ``freshness``, + ``volume``, ``consistency``. Defaults to the check's catalog dimension (see the table below; + ``validity`` for ``custom_sql``) when left unset. Only set this explicitly for a ``custom_sql`` + rule that measures something the default dimension doesn't capture. + +Built-in checks +---------------- + +Each built-in check is backed by a plain SQL expression, rendered with ``{column}`` for +column-level checks: + +.. list-table:: + :header-rows: 1 + + * - Check + - SQL expression + - Column required + - Default dimension + * - ``null_count`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)`` + - yes + - ``completeness`` + * - ``null_ratio`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)`` + - yes + - ``completeness`` + * - ``distinct_count`` + - ``COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``unique_violations`` + - ``COUNT({column}) - COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``min`` + - ``MIN({column})`` + - yes + - ``validity`` + * - ``max`` + - ``MAX({column})`` + - yes + - ``validity`` + * - ``mean`` + - ``AVG({column})`` + - yes + - ``validity`` + * - ``row_count`` + - ``COUNT(*)`` + - no + - ``volume`` + +.. note:: + + ``null_ratio`` divides by ``COUNT(*)`` with no guard against an empty table (or an empty + partition, when combined with ``partition_clause``); running it against zero rows is a + division-by-zero at the database level, not a clean "not applicable" result. Guard with a + ``row_count`` rule upstream, or a ``partition_clause`` that guarantees a non-empty set. A + portable guard would wrap the denominator in ``NULLIF(COUNT(*), 0)``, but ``NULLIF`` isn't + supported by every ``DbApiHook`` (see `Supported checks and databases`_) -- this is one + instance of the general rule below: if a built-in check's SQL doesn't work against your + database, express it as ``custom_sql`` instead. + +Conditions +----------- + +A :class:`~airflow.providers.dq.rules.Condition` is the pass/fail rule applied to the observed +value, using the same grammar as the ``common.sql`` check operators: + +- ``equal_to`` -- exact match. Cannot be combined with any other comparison. +- ``greater_than`` / ``geq_to`` -- lower bound, exclusive/inclusive. +- ``less_than`` / ``leq_to`` -- upper bound, exclusive/inclusive. +- ``tolerance`` -- a percentage that widens ``equal_to`` into a range; only valid together + with ``equal_to``. + +``greater_than``/``less_than``/``geq_to``/``leq_to`` may be combined to express a range +(e.g. ``{"geq_to": 0, "leq_to": 10}``). + +``custom_sql``: the escape hatch +---------------------------------- + +Built-in checks are all single-column. The moment a rule needs to compare two columns, join +another table, or use a function the catalog doesn't cover, use ``custom_sql`` -- any SQL +statement that resolves to a single scalar, evaluated exactly like a built-in check: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +Supported checks and databases +-------------------------------- + +Built-in checks are plain SQL expressions executed through whichever ``common.sql`` +:class:`~airflow.providers.common.sql.hooks.sql.DbApiHook` your connection resolves to. +Airflow does not validate those expressions per database dialect. + +The catalog intentionally uses simple expressions that work on common relational databases +and many distributed SQL engines, but support is not guaranteed for every ``DbApiHook``. +Some hooks expose non-standard SQL layers and may reject or evaluate an expression differently. + +For example, ``null_ratio`` is currently rendered as: + +.. code-block:: text + + SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) + +This is simple and portable, but it is not guarded against an empty table or an empty +partition. A database-specific version could use a different expression, for example: + +.. code-block:: text + + CASE WHEN COUNT(*) = 0 THEN NULL + ELSE SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) END + +If a built-in check does not work for your database or you need different empty-table +semantics, use ``custom_sql`` and write the expression for your dialect. + +Loading rules from YAML +------------------------- + +Anywhere a ``RuleSet`` is accepted -- ``DQCheckOperator(ruleset=...)``, +``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.dq.assets.asset_quality` -- a path +string is accepted too, and resolved via :meth:`~airflow.providers.dq.rules.RuleSet.from_file` +at Dag-parse time. This keeps rules editable by people who don't write Python. + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/orders_ruleset.yaml + :language: yaml + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +Identity and history +---------------------- + +Each rule has a stable ``rule_uid``, a hash of the fields that define *what is being measured*: +``name`` (or ``previous_name``, if set), ``check``, ``column``, ``sql``, and ``condition``. +Everything else -- including ``severity``, ``partition_clause``, ``description``, and +``dimension`` -- can change between Dag runs without breaking the rule's execution history, +because it isn't part of the identity hash. + +Renaming a rule outright would normally start a new history under the new name; set +``previous_name`` to the old name for one deploy to carry the old identity forward instead: + +.. code-block:: python + + DQRule( + name="order_id_is_unique", # renamed from order_id_unique + previous_name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + ) diff --git a/providers/dq/docs/security.rst b/providers/dq/docs/security.rst new file mode 100644 index 0000000000000..15a0ebbb2d054 --- /dev/null +++ b/providers/dq/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/dq/provider.yaml b/providers/dq/provider.yaml new file mode 100644 index 0000000000000..b886336584a33 --- /dev/null +++ b/providers/dq/provider.yaml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-dq +name: Data Quality +description: | + ``Data Quality Provider`` + + Declarative data quality rules with durable, per-rule execution history. + Checks run through ``common.sql`` DB-API hooks; results are persisted to a + configurable results store (object storage or local files) so task, run, + and rule-level quality can be inspected over time. + +state: ready +lifecycle: incubation +source-date-epoch: 1751587200 +build-system: hatchling + +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 0.1.0 + +integrations: + - integration-name: Data Quality + external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-dq/ + how-to-guide: + - /docs/apache-airflow-providers-dq/operators.rst + tags: [software] + +operators: + - integration-name: Data Quality + python-modules: + - airflow.providers.dq.operators.dq_check + +task-decorators: + - class-name: airflow.providers.dq.decorators.dq_check.dq_check_task + name: dq_check + +config: + dq: + description: | + Configuration for the Data Quality provider results store. + options: + results_path: + description: | + Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality + results are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``. + When unset, checks still run but no history is persisted. + version_added: 0.1.0 + type: string + example: s3://data-platform/airflow-dq + default: ~ + results_conn_id: + description: | + Optional Airflow connection used to access ``results_path``. + version_added: 0.1.0 + type: string + example: aws_default + default: ~ diff --git a/providers/dq/pyproject.toml b/providers/dq/pyproject.toml new file mode 100644 index 0000000000000..1502f9e3d7804 --- /dev/null +++ b/providers/dq/pyproject.toml @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = [ + "hatchling==1.30.1", + "packaging==26.2", + "pathspec==1.1.1", + "pluggy==1.6.0", + "tomli==2.4.1; python_version < '3.11'", + "trove-classifiers==2026.6.1.19", +] +build-backend = "hatchling.build" + +[project] +name = "apache-airflow-providers-dq" +version = "0.1.0" +description = "Provider package apache-airflow-providers-dq for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "dq", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=3.0.0", + "apache-airflow-providers-common-compat>=1.15.0", + "apache-airflow-providers-common-sql>=2.0.0", + "pydantic>=2.11.0", + "pyyaml>=6.0.2", +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + "apache-airflow-providers-common-compat", + "apache-airflow-providers-common-sql", + # Additional devel dependencies (do not remove this line and add extra development dependencies) + "apache-airflow-providers-common-sql", +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dq/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dq/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.dq.get_provider_info:get_provider_info" + +[tool.hatch.version] +path = "src/airflow/providers/dq/__init__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "docs", + "src/airflow/providers/dq", + "tests", + "NOTICE" +] +exclude = [ + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", +] + +[tool.hatch.build.targets.custom] +path = "./hatch_build.py" + +artifacts = [ +] + +[tool.hatch.build.targets.wheel] +packages = ['src/airflow'] +artifacts = [ +] +exclude = [ + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", +] diff --git a/providers/dq/src/airflow/__init__.py b/providers/dq/src/airflow/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/src/airflow/providers/__init__.py b/providers/dq/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/src/airflow/providers/dq/__init__.py b/providers/dq/src/airflow/providers/dq/__init__.py new file mode 100644 index 0000000000000..4fd31dec8bc1a --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/__init__.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "3.0.0" +): + raise RuntimeError(f"The package `apache-airflow-providers-dq:{__version__}` needs Apache Airflow 3.0.0+") diff --git a/providers/dq/src/airflow/providers/dq/assets.py b/providers/dq/src/airflow/providers/dq/assets.py new file mode 100644 index 0000000000000..513587a68d9b4 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/assets.py @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Asset-level data quality declarations. + +Quality configuration lives inside ``Asset.extra`` under the ``airflow.dq`` key, so it is +serialized with the Dag and needs no Airflow core changes. The rules travel with the asset +definition instead of being scattered across the Dags that check it. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from airflow.providers.dq.exceptions import DQRuleValidationError +from airflow.providers.dq.rules import RuleSet +from airflow.sdk import task + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from airflow.sdk import Asset + from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult + +log = logging.getLogger(__name__) + +DQ_EXTRA_KEY = "airflow.dq" +DQ_RESULT_EXTRA_KEY = "airflow.dq.result" + + +def asset_quality( + asset: Asset, + *, + ruleset: RuleSet | dict[str, Any] | str, + conn_id: str | None = None, + table: str | None = None, +) -> Asset: + """ + Attach data quality configuration to an asset, returning the same asset. + + :param asset: The asset the rules describe. + :param ruleset: A :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file (resolved eagerly, at Dag-parse time). + :param conn_id: Default connection a check operator should use for this asset. + :param table: Default table to check; falls back to the asset name when unset. + + Usage:: + + orders = asset_quality( + Asset("orders", uri="postgres://warehouse/analytics/orders"), + ruleset=rules, + conn_id="warehouse", + table="analytics.orders", + ) + check = DQCheckOperator(task_id="dq", asset=orders) + """ + if isinstance(ruleset, str): + ruleset = RuleSet.from_file(ruleset) + elif isinstance(ruleset, dict): + ruleset = RuleSet.from_dict(ruleset) + config: dict[str, Any] = {"ruleset": ruleset.to_dict()} + if conn_id: + config["conn_id"] = conn_id + if table: + config["table"] = table + asset.extra[DQ_EXTRA_KEY] = config + return asset + + +def get_asset_quality_config(asset: Asset) -> dict[str, Any] | None: + """Return the raw ``airflow.dq`` config attached to an asset, if any.""" + config = asset.extra.get(DQ_EXTRA_KEY) + if not isinstance(config, dict): + return None + return config + + +def get_asset_ruleset(asset: Asset) -> RuleSet: + """Return the ruleset attached to an asset, raising when none is attached.""" + config = get_asset_quality_config(asset) + if not config or "ruleset" not in config: + raise DQRuleValidationError( + f"Asset {asset.name!r} has no data quality config; attach one with asset_quality()" + ) + return RuleSet.from_dict(config["ruleset"]) + + +def _quality_score_passes( + asset: Asset, + min_score: float, + triggering_asset_events: Mapping[Asset, Sequence[AssetEventDagRunReferenceResult]], +) -> bool: + """Pure decision logic behind :func:`require_quality`, kept separate so it is testable without a Dag.""" + events = triggering_asset_events.get(asset, []) + if not events: + log.warning( + "require_quality(%s): no triggering event for this run; skipping downstream tasks", + asset.name, + ) + return False + + summary = events[-1].extra.get(DQ_RESULT_EXTRA_KEY) + score = summary.get("score") if isinstance(summary, dict) else None + if not isinstance(score, (int, float)) or isinstance(score, bool): + log.warning( + "require_quality(%s): triggering event has no data quality summary; skipping downstream tasks", + asset.name, + ) + return False + + if score < min_score: + log.warning( + "require_quality(%s): score %s below required minimum %s; skipping downstream tasks", + asset.name, + score, + min_score, + ) + return False + + return True + + +def require_quality( + asset: Asset, + *, + min_score: float, + task_id: str | None = None, +) -> Any: + """ + Gate a Dag run on the data quality score attached to one of its triggering asset events. + + Reads the summary a :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` + attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dq.result"]`` + (see :func:`asset_quality`), and short-circuits the run — skipping every downstream task + — when that summary is missing or its score is below ``min_score``. Call it inside a Dag + scheduled by ``asset``:: + + with DAG("orders_consumer", schedule=orders) as consumer: + start = require_quality(orders, min_score=0.95) + start >> process_orders() + + :param asset: The asset whose triggering event carries the quality summary. + :param min_score: Minimum required score in ``[0, 1]``; the run proceeds only when the + triggering event's score is at least this value. + :param task_id: Task id for the generated gate task. Defaults to + ``f"require_quality_{asset.name}"`` so gating on several assets in one Dag doesn't + collide on task id. + """ + if not 0 <= min_score <= 1: + raise ValueError(f"min_score must be between 0 and 1, got {min_score!r}") + gate_task_id = task_id or f"require_quality_{asset.name}" + + @task.short_circuit(task_id=gate_task_id) + def _require_quality(**context: Any) -> bool: + return _quality_score_passes(asset, min_score, context["triggering_asset_events"]) + + return _require_quality() diff --git a/providers/dq/src/airflow/providers/dq/backends/__init__.py b/providers/dq/src/airflow/providers/dq/backends/__init__.py new file mode 100644 index 0000000000000..7fe7bade843a5 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/backends/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend + +__all__ = ["ObjectStorageResultsBackend", "get_backend_from_config"] + + +def get_backend_from_config() -> ObjectStorageResultsBackend | None: + """Build the results backend configured under ``[dq]``, or ``None`` when not configured.""" + from airflow.providers.common.compat.sdk import conf + + results_path = conf.get("dq", "results_path", fallback=None) + if not results_path: + return None + conn_id = conf.get("dq", "results_conn_id", fallback=None) + return ObjectStorageResultsBackend(results_path=results_path, conn_id=conn_id or None) diff --git a/providers/dq/src/airflow/providers/dq/backends/object_storage.py b/providers/dq/src/airflow/providers/dq/backends/object_storage.py new file mode 100644 index 0000000000000..c43978a03dbeb --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/backends/object_storage.py @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Object-storage results backend. + +Each DQ check writes a keyed JSON document plus read indexes optimized for the UI: + + runs/by_task/dag_id=/task_id=/date=<2026-07-04>/.json + Canonical run record: ``{"run": ..., "results": [...], "summary": ...}``. + + runs/by_task_instance/dag_id=/task_id=/__.json + Latest result for a task-instance page. Last write wins across retries. + + rules/by_rule/rule_uid=/__.json + One rule result plus run context: ``{"run": ..., "result": ...}``. + + rules/by_task_rule/dag_id=/task_id=/rule_uid=/__.json + Same payload, scoped for task-level rule history views. + +The duplicate files are intentional read indexes: DQ tasks write once, while the UI reads +many times. Keeping these indexes avoids scanning all task runs for common UI views. +""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any + +from airflow.providers.dq.results import DQRun, RuleResult, build_summary +from airflow.sdk import ObjectStoragePath + +log = logging.getLogger(__name__) + + +class ObjectStorageResultsBackend: + """Persist DQ results as JSON files via ``ObjectStoragePath``.""" + + def __init__(self, results_path: str, conn_id: str | None = None) -> None: + self.root = ObjectStoragePath(results_path, conn_id=conn_id) + + def write_run(self, run: DQRun, results: list[RuleResult]) -> None: + timestamp = run.started_at or datetime.now(tz=timezone.utc).isoformat() + payload = self._build_run_payload(run, results) + + self._write_run_file(run, timestamp[:10], payload) + self._write_task_instance_index(run, payload) + self._write_rule_indexes(run, results, timestamp) + + def read_task_rule_history( + self, dag_id: str, task_id: str, rule_uid: str, limit: int = 100, before: str | None = None + ) -> dict[str, Any]: + """Return recent results for one rule produced by one task, newest first.""" + rule_dir = ( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"rule_uid={rule_uid}" + ) + return self._read_rule_history_dir(rule_dir, limit, before) + + def read_task_runs( + self, dag_id: str, task_id: str, limit: int = 50, before: str | None = None + ) -> dict[str, Any]: + """ + Return recent data quality runs for one task, newest first. + + Returns ``{"items": [...], "next_cursor": ...}``. ``before`` is the opaque + ``next_cursor`` from the previous page, so "load more" only reads runs it hasn't + shown yet. ``next_cursor`` is set by reading one extra entry past ``limit``: cheap + on every page except the last one, where there's no way to confirm history is + exhausted without walking to the end. + + ``date=`` partition names sort correctly as plain strings, so directories are walked + newest-first and scanning stops as soon as ``limit + 1`` matching runs have been + collected — a task with years of history doesn't pay for a full scan on every page. + """ + task_dir = self.root / "runs" / "by_task" / f"dag_id={dag_id}" / f"task_id={task_id}" + if not task_dir.exists(): + return {"items": [], "next_cursor": None} + + date_dirs = sorted((path for path in task_dir.iterdir() if path.is_dir()), reverse=True) + runs = [] + for date_dir in date_dirs: + for path in date_dir.iterdir(): + if not path.name.endswith(".json"): + continue + payload = self._read_json(path) + if payload is None: + continue + cursor = self._get_run_payload_cursor(payload) + if before is not None and cursor >= before: + continue + runs.append(payload) + if len(runs) > limit: + break + + ordered = sorted(runs, key=self._get_run_payload_cursor, reverse=True) + page = ordered[:limit] + next_cursor = self._get_run_payload_cursor(page[-1]) if len(ordered) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + def read_by_task_instance( + self, dag_id: str, task_id: str, run_id: str, map_index: int = -1 + ) -> dict[str, Any]: + """Read the latest run for one task instance as ``{"run": ..., "results": ..., "summary": ...}``.""" + path = ( + self.root + / "runs" + / "by_task_instance" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"{self._get_safe_key(run_id)}__{map_index}.json" + ) + return self._read_json_or_raise(path) + + def _write_run_file(self, run: DQRun, date_part: str, payload: dict[str, Any]) -> None: + run_dir = ( + self.root + / "runs" + / "by_task" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"date={date_part}" + ) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / f"{run.run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _write_task_instance_index(self, run: DQRun, payload: dict[str, Any]) -> None: + ti_dir = self.root / "runs" / "by_task_instance" / f"dag_id={run.dag_id}" / f"task_id={run.task_id}" + ti_dir.mkdir(parents=True, exist_ok=True) + (ti_dir / f"{self._get_safe_key(run.run_id)}__{run.map_index}.json").write_text( + json.dumps(payload, default=str) + ) + + def _write_rule_indexes(self, run: DQRun, results: list[RuleResult], timestamp: str) -> None: + run_context = self._build_run_context(run) + compact_ts = self._get_safe_key(timestamp) + for result in results: + payload = {"run": run_context, "result": result.to_dict()} + self._write_rule_index( + self.root / "rules" / "by_rule" / f"rule_uid={result.rule_uid}", + compact_ts, + run.run_uid, + payload, + ) + self._write_rule_index( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"rule_uid={result.rule_uid}", + compact_ts, + run.run_uid, + payload, + ) + + def _write_rule_index( + self, rule_dir: ObjectStoragePath, compact_ts: str, run_uid: str, payload: dict[str, Any] + ) -> None: + rule_dir.mkdir(parents=True, exist_ok=True) + (rule_dir / f"{compact_ts}__{run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _read_rule_history_dir( + self, rule_dir: ObjectStoragePath, limit: int, before: str | None = None + ) -> dict[str, Any]: + """ + Read rule-result records newest-first, as ``{"items": [...], "next_cursor": ...}``. + + Reads one entry past ``limit`` to determine ``next_cursor``: cheap on every page + except the last one, where confirming there's nothing older means reading to the end. + """ + if not rule_dir.exists(): + return {"items": [], "next_cursor": None} + + history: list[dict[str, Any]] = [] + for path in sorted(rule_dir.iterdir(), key=lambda p: p.name, reverse=True): + payload = self._read_json(path) + if payload is None: + continue + record = {**payload["result"], "run": payload["run"]} + cursor = self._get_rule_history_cursor(record) + if before is not None and cursor >= before: + continue + history.append(record) + if len(history) > limit: + break + + page = history[:limit] + next_cursor = self._get_rule_history_cursor(page[-1]) if len(history) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + @staticmethod + def _get_safe_key(value: str) -> str: + """Sanitize a value (for example an Airflow ``run_id``) for an object key segment.""" + return value.replace("/", "_").replace(":", "_").replace("+", "_") + + @staticmethod + def _build_run_payload(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + result_records = [result.to_dict() for result in results] + return { + "run": run.to_dict(), + "results": result_records, + "summary": build_summary(run, results), + } + + @staticmethod + def _build_run_context(run: DQRun) -> dict[str, Any]: + return { + "run_uid": run.run_uid, + "dag_id": run.dag_id, + "task_id": run.task_id, + "run_id": run.run_id, + "map_index": run.map_index, + "started_at": run.started_at, + "table_ref": run.table_ref, + } + + @staticmethod + def _get_run_payload_cursor(payload: dict[str, Any]) -> str: + run = payload["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _get_rule_history_cursor(record: dict[str, Any]) -> str: + run = record["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _build_cursor(started_at: str | None, run_uid: str | None) -> str: + return f"{started_at or ''}|{run_uid or ''}" + + def _read_json_or_raise(self, path: ObjectStoragePath) -> dict[str, Any]: + return json.loads(path.read_text()) + + def _read_json(self, path: ObjectStoragePath) -> dict[str, Any] | None: + try: + return self._read_json_or_raise(path) + except (OSError, json.JSONDecodeError): + log.warning("Skipping unreadable DQ result file %s", path) + return None diff --git a/providers/dq/src/airflow/providers/dq/decorators/__init__.py b/providers/dq/src/airflow/providers/dq/decorators/__init__.py new file mode 100644 index 0000000000000..21d298ede6ed3 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/decorators/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations diff --git a/providers/dq/src/airflow/providers/dq/decorators/dq_check.py b/providers/dq/src/airflow/providers/dq/decorators/dq_check.py new file mode 100644 index 0000000000000..0385187b1d39d --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/decorators/dq_check.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""TaskFlow decorator for :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator`.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence +from typing import TYPE_CHECKING, Any, ClassVar + +from airflow.providers.common.compat.sdk import ( + DecoratedOperator, + context_merge, + determine_kwargs, + task_decorator_factory, +) +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import RuleSet +from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION + +if TYPE_CHECKING: + from airflow.sdk import Context + from airflow.sdk.bases.decorator import TaskDecorator + + +class _DQCheckDecoratedOperator(DecoratedOperator, DQCheckOperator): + """ + Wraps a callable that optionally returns a runtime ruleset for a data quality check. + + :param python_callable: A reference to a callable returning ``None`` or a ruleset for this run. + :param op_args: Positional arguments for the callable. + :param op_kwargs: Keyword arguments for the callable. + """ + + template_fields: Sequence[str] = ( + *DecoratedOperator.template_fields, + *DQCheckOperator.template_fields, + ) + template_fields_renderers: ClassVar[dict[str, str]] = { + **DecoratedOperator.template_fields_renderers, + } + + custom_operator_name = "@task.dq_check" + + def __init__( + self, + *, + python_callable: Callable, + op_args: Collection[Any] | None = None, + op_kwargs: Mapping[str, Any] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("ruleset", SET_DURING_EXECUTION) + super().__init__(python_callable=python_callable, op_args=op_args, op_kwargs=op_kwargs, **kwargs) + + def execute(self, context: Context) -> Any: + context_merge(context, self.op_kwargs) + kwargs = determine_kwargs(self.python_callable, self.op_args, context) + ruleset = self.python_callable(*self.op_args, **kwargs) + if ruleset is not None: + if not isinstance(ruleset, (RuleSet, dict, str)): + raise TypeError( + f"{self.custom_operator_name} function must return a RuleSet, dict, path, or None, " + f"got {type(ruleset).__name__}" + ) + self.ruleset = ruleset + return DQCheckOperator.execute(self, context) + + +def dq_check_task( + python_callable: Callable | None = None, + **kwargs, +) -> TaskDecorator: + """ + Turn a function into a data quality check task. + + ``ruleset`` may be passed as a decorator argument when known at Dag-parse time, or returned + by the decorated function at runtime. ``table`` or ``asset`` are passed through ``kwargs`` + exactly as on :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator`. The + function itself is optional plumbing: return ``None`` to run the check as declared, or a + ruleset to use for this run. + + Usage:: + + @task.dq_check(conn_id="warehouse", ruleset=rules) + def orders_quality(ds=None): + return None + + :param python_callable: Function to decorate. + """ + return task_decorator_factory( + python_callable=python_callable, + decorated_operator_class=_DQCheckDecoratedOperator, + **kwargs, + ) diff --git a/providers/dq/src/airflow/providers/dq/engines/__init__.py b/providers/dq/src/airflow/providers/dq/engines/__init__.py new file mode 100644 index 0000000000000..98a6f6fe9ba80 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/engines/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.engines.sql import Observation, SQLDQEngine + +__all__ = ["Observation", "SQLDQEngine"] diff --git a/providers/dq/src/airflow/providers/dq/engines/sql.py b/providers/dq/src/airflow/providers/dq/engines/sql.py new file mode 100644 index 0000000000000..43c2f9579f8df --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/engines/sql.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""SQL execution engine: compiles a ruleset into check queries run through a DB-API hook.""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from airflow.providers.dq.rules import CUSTOM_SQL_CHECK +from airflow.providers.dq.rules.checks import CHECK_SPECS + +if TYPE_CHECKING: + from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.providers.dq.rules import DQRule, RuleSet + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Observation: + """Raw value a rule observed, before its condition is evaluated.""" + + rule: DQRule + observed_value: Any = None + duration_ms: float | None = None + error_message: str | None = None + sql: str | None = None + + +class SQLDQEngine: + """ + Runs built-in checks as a single UNION ALL query and custom SQL rules individually. + + Table, column, and partition-clause values come from the Dag author and are interpolated + into SQL the same way the ``common.sql`` check operators do — they are trusted input. + """ + + check_sql_template = "SELECT '{rule_uid}' AS rule_uid, {expression} AS observed FROM {table}{where}" + + def __init__(self, hook: DbApiHook) -> None: + self.hook = hook + + def measure(self, ruleset: RuleSet, table: str, partition_clause: str | None = None) -> list[Observation]: + builtin_rules = [rule for rule in ruleset.rules if rule.check != CUSTOM_SQL_CHECK] + custom_rules = [rule for rule in ruleset.rules if rule.check == CUSTOM_SQL_CHECK] + + observations = [] + if builtin_rules: + observations.extend(self._measure_builtin(builtin_rules, table, partition_clause)) + for rule in custom_rules: + observations.append(self._measure_custom(rule, table)) + return observations + + def build_batch_sql(self, rules: list[DQRule], table: str, partition_clause: str | None) -> str: + return " UNION ALL ".join(self.build_rule_sql(rule, table, partition_clause) for rule in rules) + + def build_rule_sql(self, rule: DQRule, table: str, partition_clause: str | None = None) -> str: + """Build the SQL used to measure one built-in rule.""" + expression = CHECK_SPECS[rule.check].expression.format(column=rule.column) + predicates = [p for p in (partition_clause, rule.partition_clause) if p] + where = f" WHERE {' AND '.join(predicates)}" if predicates else "" + return self.check_sql_template.format( + rule_uid=rule.rule_uid, expression=expression, table=table, where=where + ) + + def _measure_builtin( + self, rules: list[DQRule], table: str, partition_clause: str | None + ) -> list[Observation]: + sql = self.build_batch_sql(rules, table, partition_clause) + log.info("Running %d built-in checks against %s", len(rules), table) + started = time.monotonic() + try: + records = self.hook.get_records(sql) + except Exception as e: + elapsed_ms = (time.monotonic() - started) * 1000 + log.exception("Check query failed for table %s", table) + return [ + Observation( + rule=rule, + duration_ms=elapsed_ms, + error_message=str(e), + sql=self.build_rule_sql(rule, table, partition_clause), + ) + for rule in rules + ] + elapsed_ms = (time.monotonic() - started) * 1000 + observed_by_uid = {str(row[0]): row[1] for row in records or []} + return [ + Observation( + rule=rule, + observed_value=observed_by_uid.get(rule.rule_uid), + duration_ms=elapsed_ms, + error_message=None if rule.rule_uid in observed_by_uid else "No result returned for rule", + sql=self.build_rule_sql(rule, table, partition_clause), + ) + for rule in rules + ] + + def _measure_custom(self, rule: DQRule, table: str) -> Observation: + if rule.sql is None: + raise ValueError(f"Rule {rule.name!r} has no SQL to execute") + sql = rule.sql.replace("{table}", table) + log.info("Running custom SQL check %s", rule.name) + started = time.monotonic() + try: + row = self.hook.get_first(sql) + except Exception as e: + log.exception("Custom SQL check %s failed", rule.name) + return Observation( + rule=rule, + duration_ms=(time.monotonic() - started) * 1000, + error_message=str(e), + sql=sql, + ) + elapsed_ms = (time.monotonic() - started) * 1000 + if row is None: + return Observation( + rule=rule, duration_ms=elapsed_ms, error_message="Query returned no rows", sql=sql + ) + return Observation(rule=rule, observed_value=row[0], duration_ms=elapsed_ms, sql=sql) diff --git a/providers/dq/src/airflow/providers/dq/example_dags/__init__.py b/providers/dq/src/airflow/providers/dq/example_dags/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py new file mode 100644 index 0000000000000..f07f746be9849 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``custom_sql`` for checks the built-in catalog can't express. + +Built-in checks (``null_count``, ``min``, ``max``, ...) are all single-column. The moment a +rule needs to compare two columns, join against another table, or use a SQL function the +built-in catalog doesn't cover (or that your database's ``DbApiHook`` doesn't support -- see +"Supported checks and databases" in the provider docs), ``custom_sql`` is the escape hatch: +any statement that resolves to a single scalar, evaluated the same way as a built-in check. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.sdk import DAG + +DAG_ID = "example_dq_check_custom_sql" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_custom_sql_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_custom_sql] +custom_sql_ruleset = RuleSet( + name="orders_custom_sql", + rules=( + DQRule( + name="shipped_after_ordered", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string + # placeholder -- a cross-column comparison no single-column built-in can express. + sql="SELECT COUNT(*) FROM {table} WHERE shipped_at < ordered_at", + condition={"equal_to": 0}, + ), + DQRule( + name="high_value_order_ratio", + check="custom_sql", + sql=( + "SELECT CAST(SUM(CASE WHEN amount > 100 THEN 1 ELSE 0 END) AS REAL) / COUNT(*) FROM {table}" + ), + condition={"leq_to": 0.5}, + severity=Severity.WARN, + ), + ), +) +# [END howto_operator_dq_check_custom_sql] + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + amount REAL, + ordered_at TEXT, + shipped_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ordered_at, shipped_at) VALUES + (1, 42.0, '2026-07-01', '2026-07-02'), + (2, 150.0, '2026-07-01', '2026-07-03'), + (3, 15.5, '2026-07-02', '2026-07-02'); + """, + ) + + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=custom_sql_ruleset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py new file mode 100644 index 0000000000000..2a4b782539a11 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``@task.dq_check`` with a runtime ruleset. + +``table`` (or ``asset``) is declared as a decorator argument, exactly like the plain operator. +``ruleset`` may be declared at Dag-parse time, or returned by the decorated function at +execution time. This is useful when an upstream task, variable, or generated value decides which +ruleset to run. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk import DAG, task + +DAG_ID = "example_dq_check_decorator_dynamic" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_dynamic_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +orders_ruleset = RuleSet( + name="orders_dynamic", + rules=( + DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), + DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + ), +) + +strict_ruleset = RuleSet( + name="orders_dynamic_strict", + rules=( + *orders_ruleset.rules, + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), + ), +) + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL, ds TEXT);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ds) VALUES + (1, 10.0, '{{{{ ds }}}}'), + (2, 25.5, '{{{{ ds }}}}'), + (3, 7.25, '2020-01-01'); + """, + ) + + # [START howto_decorator_dq_check_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, ruleset=orders_ruleset, fail_on="never") + def check_with_ruleset_from_upstream(strict: bool = True): + """Swap in a stricter ruleset based on a signal only known at task-execution time.""" + return strict_ruleset if strict else None + + # [END howto_decorator_dq_check_runtime_ruleset] + + create_table >> insert_orders >> check_with_ruleset_from_upstream() diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py new file mode 100644 index 0000000000000..7ecfd743ceb91 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: generate a ``RuleSet`` with an LLM, guided by the ``dq-rule-authoring`` skill. + +Requires the optional ``apache-airflow-providers-common-ai[skills]`` package and a configured +LLM connection (``llm_conn_id``); this Dag is not registered when common.ai isn't installed. + +An LLM is asked to propose data quality rules for a table's columns. Its ``system_prompt`` +points it at the ``dq-rule-authoring`` skill shipped with this provider (via +``AgentSkillsToolset``), so it knows the exact ``RuleSet``/``DQRule`` schema, the built-in check +catalog, and the ``Condition`` grammar, instead of guessing at field or check names. +``output_type=RuleSet`` makes pydantic-ai validate -- and self-correct -- the model's output +against the real ``RuleSet`` model before the task completes, so a malformed rule never reaches +``DQCheckOperator`` in the first place. + +The generated ``RuleSet`` is then returned by ``@task.dq_check`` at runtime, since the real +ruleset doesn't exist until the LLM task runs. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.rules import RuleSet +from airflow.sdk import DAG, task + +try: + from airflow.providers.common.ai.toolsets.skills import AgentSkillsToolset +except ImportError: + AgentSkillsToolset = None # type: ignore[assignment,misc] + +DAG_ID = "example_dq_llm_generated_ruleset" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_llm_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +# The skill ships next to this Dag's package, not next to this file -- resolve relative to +# __file__ so the path holds regardless of the Dag processor's working directory. +SKILLS_DIR = Path(__file__).parent.parent / "skills" / "dq-rule-authoring" + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +if AgentSkillsToolset is not None: + with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq", "ai"], + ) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + region TEXT, + created_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, region, created_at) VALUES + (1, 101, 10.0, 'US', '2026-07-01'), + (2, 102, 25.5, 'EU', '2026-07-02'), + (3, NULL, 7.25, 'US', '2026-07-03'); + """, + ) + + # [START howto_task_dq_generate_ruleset_with_llm] + @task.llm( + llm_conn_id="pydanticai_default", + system_prompt=( + "You are a data quality engineer. Before answering, consult the " + "dq-rule-authoring skill for the exact RuleSet/DQRule schema, the built-in " + "check catalog, and the Condition grammar -- do not invent field or check names." + ), + output_type=RuleSet, + agent_params={"toolsets": [AgentSkillsToolset(sources=[str(SKILLS_DIR)])]}, + ) + def generate_ruleset(): + return ( + "Generate a RuleSet named 'orders_llm_quality' for a table with these columns: " + "order_id (integer, primary key), customer_id (integer, nullable foreign key), " + "amount (float, must be non-negative), region (short string, low-cardinality), " + "created_at (date). Include at least a not-null check on order_id, a uniqueness " + "check on order_id, and a row-count check." + ) + + # [END howto_task_dq_generate_ruleset_with_llm] + + # [START howto_decorator_dq_check_llm_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, fail_on="never") + def check_with_llm_ruleset(ruleset: RuleSet): + return ruleset + + # [END howto_decorator_dq_check_llm_runtime_ruleset] + + generated_ruleset = generate_ruleset() + checked = check_with_llm_ruleset(generated_ruleset) + + create_table >> insert_orders >> generated_ruleset >> checked diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py new file mode 100644 index 0000000000000..124156604a11e --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAGs: gate a consumer Dag on the data quality score of the asset that triggers it. + +Two Dags, wired together only through the ``dq_gated_orders`` asset: + +- ``example_dq_require_quality_producer`` checks a table and produces the asset. + :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` attaches its summary + (including the quality ``score``) to the asset event automatically because the asset carries + quality config attached with :func:`~airflow.providers.dq.assets.asset_quality`. +- ``example_dq_require_quality_consumer`` is scheduled by that asset. Its first task, built by + :func:`~airflow.providers.dq.assets.require_quality`, reads the score off the triggering + asset event and short-circuits -- skipping every downstream task -- when it's missing or + below ``min_score``. Downstream tasks never see a run triggered by a bad batch of data. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.assets import asset_quality, require_quality +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk import DAG, Asset, task + +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_gated_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_asset_quality] +gated_orders_ruleset = RuleSet( + name="gated_orders_quality", + rules=( + DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), + DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + ), +) + +gated_orders_asset = asset_quality( + Asset("dq_gated_orders", uri="file:///tmp/airflow_dq_example/gated_orders"), + ruleset=gated_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_asset_quality] + +with DAG( + dag_id="example_dq_require_quality_producer", + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as producer_dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # Producing check: no explicit outlets needed, DQCheckOperator adds the asset itself + # because it was passed via asset=. + check_orders = DQCheckOperator( + task_id="check_orders", + asset=gated_orders_asset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders + +# [START howto_require_quality] +with DAG( + dag_id="example_dq_require_quality_consumer", + schedule=gated_orders_asset, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as consumer_dag: + gate = require_quality(gated_orders_asset, min_score=0.95) + + @task + def process_orders(): + print("Processing orders -- quality gate passed.") + + gate >> process_orders() +# [END howto_require_quality] diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py new file mode 100644 index 0000000000000..25b9a6cf76abd --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: load a ruleset from a YAML file instead of declaring it in Python. + +A path string is accepted anywhere a :class:`~airflow.providers.dq.rules.RuleSet` is -- +``DQCheckOperator(ruleset=...)``, ``@task.dq_check(ruleset=...)``, and +:func:`~airflow.providers.dq.assets.asset_quality` all resolve it via +:meth:`~airflow.providers.dq.rules.RuleSet.from_file` at Dag-parse time. This keeps rules +editable by people who don't write Python -- a data steward can change ``orders_ruleset.yaml`` +without touching the Dag file. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.sdk import DAG + +DAG_ID = "example_dq_ruleset_from_yaml" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_yaml_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +RULESET_FILE = Path(__file__).parent / "orders_ruleset.yaml" + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # [START howto_operator_dq_check_ruleset_from_yaml] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=str(RULESET_FILE), + fail_on="never", + ) + # [END howto_operator_dq_check_ruleset_from_yaml] + + create_table >> insert_orders >> check_orders diff --git a/providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml b/providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml new file mode 100644 index 0000000000000..049e69543bc32 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- + +name: orders_from_yaml +rules: + - name: order_id_not_null + check: null_count + column: order_id + condition: + equal_to: 0 + dimension: completeness + - name: amount_min_ge_zero + check: min + column: amount + condition: + geq_to: 0 + - name: row_count_present + check: row_count + condition: + greater_than: 0 + severity: warn diff --git a/providers/dq/src/airflow/providers/dq/exceptions.py b/providers/dq/src/airflow/providers/dq/exceptions.py new file mode 100644 index 0000000000000..9350308d46501 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/exceptions.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +class DQRuleValidationError(ValueError): + """Raised when a rule or ruleset definition is invalid.""" + + +class DQCheckFailedError(RuntimeError): + """Raised when a data quality check fails at or above the operator's ``fail_on`` severity.""" diff --git a/providers/dq/src/airflow/providers/dq/get_provider_info.py b/providers/dq/src/airflow/providers/dq/get_provider_info.py new file mode 100644 index 0000000000000..fea5885430a63 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/get_provider_info.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-dq", + "name": "Data Quality", + "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", + "integrations": [ + { + "integration-name": "Data Quality", + "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-dq/", + "how-to-guide": ["/docs/apache-airflow-providers-dq/operators.rst"], + "tags": ["software"], + } + ], + "operators": [ + { + "integration-name": "Data Quality", + "python-modules": ["airflow.providers.dq.operators.dq_check"], + } + ], + "task-decorators": [ + {"class-name": "airflow.providers.dq.decorators.dq_check.dq_check_task", "name": "dq_check"} + ], + "config": { + "dq": { + "description": "Configuration for the Data Quality provider results store.\n", + "options": { + "results_path": { + "description": "Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality\nresults are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``.\nWhen unset, checks still run but no history is persisted.\n", + "version_added": "0.1.0", + "type": "string", + "example": "s3://data-platform/airflow-dq", + "default": None, + }, + "results_conn_id": { + "description": "Optional Airflow connection used to access ``results_path``.\n", + "version_added": "0.1.0", + "type": "string", + "example": "aws_default", + "default": None, + }, + }, + } + }, + } diff --git a/providers/dq/src/airflow/providers/dq/operators/__init__.py b/providers/dq/src/airflow/providers/dq/operators/__init__.py new file mode 100644 index 0000000000000..f1734cd8846f9 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/operators/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.operators.dq_check import DQCheckOperator + +__all__ = ["DQCheckOperator"] diff --git a/providers/dq/src/airflow/providers/dq/operators/dq_check.py b/providers/dq/src/airflow/providers/dq/operators/dq_check.py new file mode 100644 index 0000000000000..f006a54686010 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/operators/dq_check.py @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from collections.abc import Sequence +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, cast + +from airflow.providers.common.sql.operators.sql import BaseSQLOperator +from airflow.providers.dq.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config +from airflow.providers.dq.backends import get_backend_from_config +from airflow.providers.dq.engines.sql import SQLDQEngine +from airflow.providers.dq.exceptions import DQCheckFailedError +from airflow.providers.dq.results import ERROR, FAIL, PASS, WARN, DQRun, RuleResult, build_summary +from airflow.providers.dq.rules import RuleSet, describe_rule +from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION + +if TYPE_CHECKING: + from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.providers.dq.engines.sql import Observation + from airflow.providers.dq.rules.rule import Condition, Dimension + from airflow.sdk import Asset, Context + +log = logging.getLogger(__name__) + +FAIL_ON_CHOICES = ("error", "warn", "never") + + +class DQCheckOperator(BaseSQLOperator): + """ + Run a ruleset against a table and persist per-rule results to the configured results store. + + Every rule is evaluated and recorded regardless of the task outcome. Execution errors + (a check query failing) always fail the task; rule failures fail the task according to + ``fail_on``. + + :param ruleset: A :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file. Optional when ``asset`` carries one. + :param table: The table to run checks against. Optional when ``asset`` carries one + (falling back to the asset name). + :param asset: An asset decorated with :func:`~airflow.providers.dq.assets.asset_quality`. + Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments + win) and is automatically added to the task's outlets so its asset events carry + the check summary. + :param partition_clause: Predicate ANDed into every built-in check's WHERE clause, + e.g. ``"ds = '{{ ds }}'"``. + :param fail_on: Severity that fails the task: ``error`` (default — warn-severity rule + failures are recorded but don't fail), ``warn`` (any rule failure fails), or + ``never`` (only execution errors fail). + :param conn_id: Connection to the database to check (any ``DbApiHook``-compatible type). + :param database: Optional database/schema overriding the connection's default. + + Results are persisted to the backend configured under ``[dq] results_path``; when that's + unset, checks still run but no history is persisted. There is no per-operator override -- + all checks in a deployment share one results store. + """ + + template_fields: Sequence[str] = ("table", "partition_clause", *BaseSQLOperator.template_fields) + ui_color: str = "#87ceeb" + + def __init__( + self, + *, + ruleset: RuleSet | dict[str, Any] | str | None = None, + table: str | None = None, + asset: Asset | None = None, + partition_clause: str | None = None, + fail_on: str = "error", + **kwargs, + ) -> None: + if asset is not None: + config = get_asset_quality_config(asset) or {} + ruleset = ruleset if ruleset is not None else config.get("ruleset") + table = table or config.get("table") or asset.name + kwargs.setdefault("conn_id", config.get("conn_id")) + outlets = list(kwargs.get("outlets") or []) + if asset not in outlets: + outlets.append(asset) + kwargs["outlets"] = outlets + super().__init__(**kwargs) + if fail_on not in FAIL_ON_CHOICES: + raise ValueError(f"fail_on must be one of {FAIL_ON_CHOICES}, got {fail_on!r}") + if ruleset is None: + raise ValueError("ruleset is required, either directly or via an asset_quality() asset") + if not table: + raise ValueError("table is required, either directly or via an asset_quality() asset") + self.ruleset = ruleset + self.table = table + self.asset = asset + self.partition_clause = partition_clause + self.fail_on = fail_on + + def execute(self, context: Context) -> dict[str, Any]: + ruleset = self._resolve_ruleset() + started_at = datetime.now(tz=timezone.utc).isoformat() + engine = self._get_engine(self.get_db_hook()) + observations = engine.measure(ruleset, self.table, self.partition_clause) + results = [self._evaluate(observation) for observation in observations] + finished_at = datetime.now(tz=timezone.utc).isoformat() + + run = self._build_run(context, ruleset, started_at, finished_at) + summary = build_summary(run, results) + self._log_results(results, summary) + self._persist(run, results) + self._attach_to_outlet_events(context, summary) + self._raise_for_failures(results, summary) + return summary + + def _get_engine(self, hook: DbApiHook) -> SQLDQEngine: + return SQLDQEngine(hook) + + def _resolve_ruleset(self) -> RuleSet: + if self.ruleset is SET_DURING_EXECUTION: + raise ValueError("ruleset is required, either directly or returned by @task.dq_check") + if isinstance(self.ruleset, RuleSet): + return self.ruleset + if isinstance(self.ruleset, dict): + return RuleSet.from_dict(self.ruleset) + return RuleSet.from_file(self.ruleset) + + @staticmethod + def _evaluate(observation: Observation) -> RuleResult: + rule = observation.rule + condition = cast("Condition", rule.condition) + dimension = cast("Dimension", rule.dimension) + error_message = observation.error_message + if error_message is None: + try: + passed = condition.evaluate(observation.observed_value) + except (TypeError, ValueError) as e: + passed = False + error_message = f"Could not evaluate observed value {observation.observed_value!r}: {e}" + else: + passed = False + + if error_message is not None: + status = ERROR + elif passed: + status = PASS + else: + status = WARN if rule.severity == "warn" else FAIL + observed = observation.observed_value + if observed is not None and not isinstance(observed, int | float | str): + observed = str(observed) + return RuleResult( + rule_uid=rule.rule_uid, + rule_name=rule.name, + status=status, + observed_value=observed, + condition=condition.to_dict(), + dimension=dimension.value, + severity=rule.severity.value, + duration_ms=observation.duration_ms, + error_message=error_message, + description=rule.description or describe_rule(rule), + sql=observation.sql, + ) + + def _build_run(self, context: Context, ruleset: RuleSet, started_at: str, finished_at: str) -> DQRun: + ti = context["ti"] + return DQRun( + dag_id=ti.dag_id, + task_id=ti.task_id, + run_id=ti.run_id, + try_number=ti.try_number, + map_index=ti.map_index if ti.map_index is not None else -1, + ruleset_name=ruleset.name, + table_ref=self.table, + asset_names=tuple(self._outlet_asset_names()), + started_at=started_at, + finished_at=finished_at, + ) + + def _outlet_asset_names(self) -> list[str]: + names = [] + for outlet in self.outlets or []: + name = getattr(outlet, "name", None) + if name: + names.append(name) + return names + + def _persist(self, run: DQRun, results: list[RuleResult]) -> None: + backend = get_backend_from_config() + if backend is None: + self.log.info("No [dq] results_path configured; skipping results persistence") + return + # Persistence is best-effort: an unreachable results store leaves a gap in + # history but must not change the outcome of the check itself. + try: + backend.write_run(run, results) + except Exception: + self.log.exception("Failed to persist data quality results; continuing") + + def _attach_to_outlet_events(self, context: Context, summary: dict[str, Any]) -> None: + try: + outlet_events = context["outlet_events"] + except KeyError: + return + for outlet in self.outlets or []: + if getattr(outlet, "name", None): + try: + outlet_events[outlet].extra[DQ_RESULT_EXTRA_KEY] = summary + except Exception: + self.log.warning("Could not attach dq summary to outlet event for %s", outlet) + + def _log_results(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + for result in results: + self.log.info( + "Rule %s [%s/%s]: %s (observed=%s, condition=%s)", + result.rule_name, + result.dimension, + result.severity, + result.status.upper(), + result.observed_value, + result.condition, + ) + self.log.info("Data quality score: %s", summary["score"]) + + def _raise_for_failures(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + errored = [r.rule_name for r in results if r.status == ERROR] + if errored: + raise DQCheckFailedError(f"Check execution errored for rules: {errored}") + failed = [r.rule_name for r in results if r.status == FAIL] + warned = [r.rule_name for r in results if r.status == WARN] + if self.fail_on == "error" and failed: + raise DQCheckFailedError(f"Data quality rules failed: {failed} (score={summary['score']})") + if self.fail_on == "warn" and (failed or warned): + raise DQCheckFailedError( + f"Data quality rules failed: {failed + warned} (score={summary['score']})" + ) + if failed or warned: + self.log.warning("Rule failures below fail_on=%s threshold: %s", self.fail_on, failed + warned) diff --git a/providers/dq/src/airflow/providers/dq/results.py b/providers/dq/src/airflow/providers/dq/results.py new file mode 100644 index 0000000000000..5fbcf04573cc7 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/results.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Result records produced by a data quality check run — immutable facts, JSON-serializable.""" + +from __future__ import annotations + +import uuid +from dataclasses import asdict, dataclass, field +from typing import Any + +PASS = "pass" +WARN = "warn" +FAIL = "fail" +ERROR = "error" + +# Weight of a warn-severity failure in the run score, relative to an error-severity failure. +WARN_SCORE_WEIGHT = 0.25 + + +@dataclass(frozen=True) +class RuleResult: + """Outcome of evaluating one rule in one run.""" + + rule_uid: str + rule_name: str + status: str # pass | warn | fail | error + observed_value: float | str | None = None + condition: dict[str, Any] = field(default_factory=dict) + dimension: str = "validity" + severity: str = "error" + duration_ms: float | None = None + error_message: str | None = None + description: str | None = None + sql: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleResult: + return cls(**data) + + +@dataclass(frozen=True) +class DQRun: + """One execution of a ruleset by one task instance.""" + + dag_id: str + task_id: str + run_id: str + try_number: int = 1 + map_index: int = -1 + run_uid: str = field(default_factory=lambda: uuid.uuid4().hex) + ruleset_name: str | None = None + table_ref: str | None = None + asset_names: tuple[str, ...] = () + started_at: str | None = None + finished_at: str | None = None + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["asset_names"] = list(self.asset_names) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRun: + data = {**data, "asset_names": tuple(data.get("asset_names", ()))} + return cls(**data) + + +def compute_score(results: list[RuleResult]) -> float | None: + """ + Weighted pass rate for a run in [0, 1]. + + Error-severity failures (and execution errors) count fully against the score; + warn-severity failures count at ``WARN_SCORE_WEIGHT``. + """ + if not results: + return None + penalty = 0.0 + for result in results: + if result.status in (FAIL, ERROR): + penalty += 1.0 + elif result.status == WARN: + penalty += WARN_SCORE_WEIGHT + return round(1.0 - penalty / len(results), 4) + + +def build_summary(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + """Compact run summary attached to XCom and outlet asset events.""" + return { + "run_uid": run.run_uid, + "ruleset": run.ruleset_name, + "table": run.table_ref, + "score": compute_score(results), + "passed": sum(1 for r in results if r.status == PASS), + "warned": sum(1 for r in results if r.status == WARN), + "failed": sum(1 for r in results if r.status == FAIL), + "errored": sum(1 for r in results if r.status == ERROR), + "failed_rules": sorted(r.rule_name for r in results if r.status in (FAIL, ERROR)), + "warned_rules": sorted(r.rule_name for r in results if r.status == WARN), + } diff --git a/providers/dq/src/airflow/providers/dq/rules/__init__.py b/providers/dq/src/airflow/providers/dq/rules/__init__.py new file mode 100644 index 0000000000000..1536086480383 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/rules/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.rules.checks import CHECK_SPECS, CheckSpec, Dimension +from airflow.providers.dq.rules.rule import ( + CUSTOM_SQL_CHECK, + Condition, + DQRule, + RuleSet, + Severity, + describe_rule, +) + +__all__ = [ + "CHECK_SPECS", + "CUSTOM_SQL_CHECK", + "CheckSpec", + "Condition", + "DQRule", + "Dimension", + "RuleSet", + "Severity", + "describe_rule", +] diff --git a/providers/dq/src/airflow/providers/dq/rules/checks.py b/providers/dq/src/airflow/providers/dq/rules/checks.py new file mode 100644 index 0000000000000..11e73a5208f30 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/rules/checks.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Built-in check catalog: one place for each check's SQL expression and metadata.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class Dimension(str, Enum): + """Data quality dimension assigned to a rule result.""" + + COMPLETENESS = "completeness" + UNIQUENESS = "uniqueness" + VALIDITY = "validity" + FRESHNESS = "freshness" + VOLUME = "volume" + CONSISTENCY = "consistency" + + +@dataclass(frozen=True) +class CheckSpec: + """ + Metadata for one built-in check. + + :param expression: SQL expression template. Column-level checks are rendered with + ``{column}``; table-level checks (``requires_column=False``) ignore it. + :param dimension: Default value for ``DQRule.dimension`` when a rule doesn't set it + explicitly. ``DQRule.dimension`` is a settable field; this is only the fallback. + :param requires_column: Whether a rule using this check must set ``column``. + :param default_condition: Default ``DQRule.condition`` (as a dict) for a rule that doesn't + set one explicitly; ``None`` means the rule must always specify a condition. + """ + + expression: str + dimension: Dimension = Dimension.VALIDITY + requires_column: bool = True + default_condition: dict[str, Any] | None = None + + +CHECK_SPECS: dict[str, CheckSpec] = { + "null_count": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)", + dimension=Dimension.COMPLETENESS, + ), + "null_ratio": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)", + dimension=Dimension.COMPLETENESS, + ), + "distinct_count": CheckSpec( + expression="COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "unique_violations": CheckSpec( + expression="COUNT({column}) - COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "min": CheckSpec(expression="MIN({column})"), + "max": CheckSpec(expression="MAX({column})"), + "mean": CheckSpec(expression="AVG({column})"), + "row_count": CheckSpec( + expression="COUNT(*)", + dimension=Dimension.VOLUME, + requires_column=False, + ), +} diff --git a/providers/dq/src/airflow/providers/dq/rules/rule.py b/providers/dq/src/airflow/providers/dq/rules/rule.py new file mode 100644 index 0000000000000..8151be8e41143 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/rules/rule.py @@ -0,0 +1,279 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Declarative data quality rule model: rules are data, not code.""" + +from __future__ import annotations + +import hashlib +import json +from enum import Enum +from typing import Any, cast + +import yaml +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from airflow.providers.dq.exceptions import DQRuleValidationError +from airflow.providers.dq.rules.checks import CHECK_SPECS, Dimension + +CUSTOM_SQL_CHECK = "custom_sql" + + +class Severity(str, Enum): + """Severity used to decide whether a failing rule fails the task.""" + + WARN = "warn" + ERROR = "error" + + +class Condition(BaseModel): + """ + Pass/fail condition evaluated against a rule's observed value. + + Uses the same grammar as the ``common.sql`` check operators: ``equal_to``, + ``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage + ``tolerance`` that widens ``equal_to`` into a range. + + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + equal_to: float | None = None + greater_than: float | None = None + less_than: float | None = None + geq_to: float | None = None + leq_to: float | None = None + tolerance: float | None = None + + @model_validator(mode="after") + def _validate_comparisons(self) -> Condition: + comparisons = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + } + set_comparisons = {name for name, value in comparisons.items() if value is not None} + if not set_comparisons: + raise ValueError(f"Condition needs at least one comparison out of: {', '.join(comparisons)}") + if self.equal_to is not None and len(set_comparisons) > 1: + raise ValueError("equal_to cannot be combined with other comparisons") + if self.tolerance is not None and self.equal_to is None: + raise ValueError("tolerance is only supported together with equal_to") + return self + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Condition: + return cls(**data) + + def to_dict(self) -> dict[str, float]: + data = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + "tolerance": self.tolerance, + } + return {k: v for k, v in data.items() if v is not None} + + def evaluate(self, observed: Any) -> bool: + if observed is None: + return False + value = float(observed) + if self.equal_to is not None: + if self.tolerance is not None: + low = self.equal_to * (1 - self.tolerance) + high = self.equal_to * (1 + self.tolerance) + return min(low, high) <= value <= max(low, high) + return value == self.equal_to + return ( + (self.greater_than is None or value > self.greater_than) + and (self.less_than is None or value < self.less_than) + and (self.geq_to is None or value >= self.geq_to) + and (self.leq_to is None or value <= self.leq_to) + ) + + +class DQRule(BaseModel): + """ + A single, named data quality rule. + + :param name: Rule name, unique within its ruleset. + :param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in + checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``; + see "Supported checks and databases" in the provider docs. Use ``custom_sql`` if a + built-in check doesn't fit your database's dialect. + :param condition: Pass condition for the observed value (``Condition`` or its dict form). + Optional only for checks whose ``CheckSpec.default_condition`` is set; every current + built-in check requires one explicitly. + :param column: Target column; required for column-level built-in checks. + :param sql: SQL statement returning a single scalar; required for ``custom_sql``. + May reference the target table as ``{table}``. + :param severity: ``error`` (fails the task by default) or ``warn`` (recorded only). + :param partition_clause: Extra predicate ANDed into the check's WHERE clause. + :param previous_name: Set when renaming a rule, to keep its history continuous. + :param description: Human-readable description shown in results and the UI. When omitted, + the provider generates a short default description from the rule and condition. + + Invalid input raises pydantic's own :class:`~pydantic.ValidationError`. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + check: str = Field(description="One of the built-in checks, or custom_sql.") + condition: Condition | dict[str, Any] | None = None + column: str | None = None + sql: str | None = None + severity: Severity = Severity.ERROR + partition_clause: str | None = None + previous_name: str | None = None + description: str | None = Field( + default=None, + description=( + "Human-readable description shown in results and the UI. When omitted, the provider " + "generates a short default description from the rule and condition." + ), + ) + dimension: Dimension | None = Field( + default=None, + description=( + "Data quality dimension recorded for this rule's results. Defaults to the check's " + "catalog dimension (validity for custom_sql) when not set explicitly." + ), + ) + + @model_validator(mode="after") + def _validate(self) -> DQRule: + if not self.name: + raise ValueError("Rule name cannot be empty") + catalog_spec = None + if self.check == CUSTOM_SQL_CHECK: + if not self.sql: + raise ValueError(f"Rule {self.name!r}: custom_sql check requires 'sql'") + else: + catalog_spec = CHECK_SPECS.get(self.check) + if catalog_spec is None: + supported = sorted([*CHECK_SPECS, CUSTOM_SQL_CHECK]) + raise ValueError(f"Rule {self.name!r}: unknown check {self.check!r}; supported: {supported}") + if self.sql: + raise ValueError(f"Rule {self.name!r}: 'sql' is only valid with custom_sql") + if catalog_spec.requires_column and not self.column: + raise ValueError(f"Rule {self.name!r}: check {self.check!r} requires 'column'") + if self.condition is None and catalog_spec.default_condition is not None: + object.__setattr__(self, "condition", Condition.from_dict(catalog_spec.default_condition)) + if self.condition is None: + raise ValueError(f"Rule {self.name!r}: condition is required for check {self.check!r}") + if self.dimension is None: + object.__setattr__( + self, "dimension", catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + ) + return self + + @property + def rule_uid(self) -> str: + """Stable identity across runs: survives severity/dimension tweaks and Dag refactors.""" + condition = cast("Condition", self.condition) + identity = { + "name": self.previous_name or self.name, + "check": self.check, + "column": self.column, + "sql": self.sql, + "condition": condition.to_dict(), + } + digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + return digest[:16] + + def to_dict(self) -> dict[str, Any]: + condition = cast("Condition", self.condition) + data: dict[str, Any] = { + "name": self.name, + "check": self.check, + "condition": condition.to_dict(), + "severity": self.severity.value, + } + for optional in ("column", "sql", "partition_clause", "previous_name", "description"): + value = getattr(self, optional) + if value is not None: + data[optional] = value + # Only emit dimension when it overrides the check's catalog default, so a rule that + # never set one explicitly round-trips through to_dict()/from_dict() unchanged. + catalog_spec = CHECK_SPECS.get(self.check) + default_dimension = catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + if self.dimension != default_dimension: + data["dimension"] = cast("Dimension", self.dimension).value + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRule: + return cls(**data) + + +def describe_rule(rule: DQRule) -> str: + """Return a default human-readable description for a rule.""" + condition = cast("Condition", rule.condition).to_dict() + subject = rule.column or rule.name.replace("_", " ") + if "equal_to" in condition: + return f"{subject} should equal {condition['equal_to']}" + if "geq_to" in condition: + return f"{subject} should be greater than or equal to {condition['geq_to']}" + if "greater_than" in condition: + return f"{subject} should be greater than {condition['greater_than']}" + if "leq_to" in condition: + return f"{subject} should be less than or equal to {condition['leq_to']}" + if "less_than" in condition: + return f"{subject} should be less than {condition['less_than']}" + return subject + + +class RuleSet(BaseModel): + """A named collection of rules, typically attached to one table or asset.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + rules: tuple[DQRule, ...] = () + + @model_validator(mode="after") + def _validate(self) -> RuleSet: + if not self.name: + raise ValueError("RuleSet name cannot be empty") + names = [rule.name for rule in self.rules] + duplicates = {name for name in names if names.count(name) > 1} + if duplicates: + raise ValueError(f"Duplicate rule names in ruleset {self.name!r}: {sorted(duplicates)}") + return self + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "rules": [rule.to_dict() for rule in self.rules]} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleSet: + rules = [ + rule if isinstance(rule, DQRule) else DQRule.from_dict(rule) for rule in data.get("rules", []) + ] + return cls(name=data.get("name", ""), rules=tuple(rules)) + + @classmethod + def from_file(cls, path: str) -> RuleSet: + """Load a ruleset from a YAML (or JSON, being a YAML subset) file.""" + with open(path) as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise DQRuleValidationError(f"Ruleset file {path!r} must contain a mapping at top level") + return cls.from_dict(data) diff --git a/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md new file mode 100644 index 0000000000000..f7a7693d49e41 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md @@ -0,0 +1,174 @@ +--- +name: dq-rule-authoring +description: "Generate a RuleSet/DQRule JSON payload for Apache Airflow's dq provider from a table's column definitions. Use this whenever asked to write, generate, or suggest data quality rules, checks, or a ruleset for a table or dataset -- especially when the output is structured JSON handed to DQCheckOperator, @task.dq_check, asset_quality(), or RuleSet.from_file()." +--- + + + +# dq rule authoring + +You are producing a **RuleSet**: a JSON object naming a table's data quality checks. This +document is the schema and the full list of valid values -- do not guess field names or check +names, and do not invent checks that aren't in the catalog below. + +## Output shape + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + } + ] +} +``` + +`name` (ruleset name) and `rules` (array) are the only top-level keys. Every rule name must be +unique within the ruleset. + +## `DQRule` fields + +| Field | Required | Notes | +|---|---|---| +| `name` | yes | Unique within the ruleset. Shows up in data quality results and logs. | +| `check` | yes | One of the catalog names below, or `"custom_sql"`. Exact string match -- there is no fuzzy matching. | +| `condition` | yes* | See `Condition` grammar below. *Every catalog check currently requires one explicitly (no defaults) -- always include it. | +| `column` | check-dependent | Required for every catalog check except `row_count`. Not used with `custom_sql`. | +| `sql` | `custom_sql` only | A SQL statement returning a single scalar. May reference the checked table as `{table}`. Invalid together with any catalog check. | +| `severity` | no | `"error"` (default) or `"warn"`. `"error"` fails the task on a failing rule (subject to the operator's `fail_on`); `"warn"` only records it. | +| `partition_clause` | no | Extra SQL predicate ANDed into this rule's `WHERE` clause, e.g. `"region = 'EU'"`. | +| `previous_name` | no | Only set when told a rule is being renamed, to keep its history continuous. | +| `description` | no | Human-readable text shown in data quality results. Use it when a clear business meaning is known; otherwise omit it and let Airflow generate a default description. | +| `dimension` | no | One of `completeness`, `uniqueness`, `validity`, `freshness`, `volume`, `consistency`. Defaults to the check's catalog dimension (`validity` for `custom_sql`) -- only set this explicitly when a `custom_sql` rule measures something the default doesn't capture (e.g. a freshness check written as `custom_sql` should set `"dimension": "freshness"`). Leave it unset for every built-in check. | + +**Do not add any key not listed above.** The schema rejects unrecognized fields. + +## Built-in check catalog + +| `check` | SQL expression | needs `column` | typical use | +|---|---|---|---| +| `null_count` | `SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)` | yes | count of nulls in a column | +| `null_ratio` | `SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)` | yes | fraction of nulls, 0.0-1.0 | +| `distinct_count` | `COUNT(DISTINCT {column})` | yes | number of distinct values | +| `unique_violations` | `COUNT({column}) - COUNT(DISTINCT {column})` | yes | 0 means the column is fully unique | +| `min` | `MIN({column})` | yes | minimum value | +| `max` | `MAX({column})` | yes | maximum value | +| `mean` | `AVG({column})` | yes | average value | +| `row_count` | `COUNT(*)` | no | total row count -- omit `column` entirely | + +These are the *only* built-in check names. Anything else -- comparing two columns, joining +another table, checking a format/regex, freshness against `now()`, referential integrity -- must +be written as `custom_sql`. + +## `Condition` grammar + +`condition` is an object with these optional numeric keys; at least one is required: + +- `equal_to` -- exact match. Cannot be combined with any other key below. +- `greater_than` / `geq_to` -- lower bound, exclusive / inclusive. +- `less_than` / `leq_to` -- upper bound, exclusive / inclusive. +- `tolerance` -- a fraction (e.g. `0.1` for 10%) that widens `equal_to` into a range. Only valid + together with `equal_to`. + +Combine `greater_than`/`less_than`/`geq_to`/`leq_to` freely to express a range, e.g. +`{"geq_to": 0, "leq_to": 100}`. Never combine `equal_to` with another comparison key (other than +`tolerance`). + +## `custom_sql`: when a catalog check doesn't fit + +Use `check: "custom_sql"` with a `sql` statement resolving to a single scalar whenever: + +- The check needs more than one column (e.g. `end_date >= start_date`), a join, or a subquery. +- A catalog expression doesn't run correctly against the target database's SQL dialect (for + example, some engines don't support `NULLIF`, or evaluate `CASE`/`COUNT DISTINCT` + differently) -- write the equivalent expression for that dialect instead. +- The check is conceptually one of the checks above but needs different null/empty-table + semantics than the catalog expression provides. + +Reference the table being checked as `{table}` in the SQL: + +```json +{ + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE order_date > CURRENT_DATE", + "condition": {"equal_to": 0} +} +``` + +## Worked example + +Given these column definitions for an `orders` table -- +`order_id` (integer, primary key), `customer_id` (integer, nullable foreign key), +`amount` (decimal, must be non-negative), `region` (string, low-cardinality), +`created_at` (timestamp) -- a reasonable ruleset: + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "order_id_unique", + "check": "unique_violations", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "customer_id_null_ratio_low", + "check": "null_ratio", + "column": "customer_id", + "condition": {"leq_to": 0.05}, + "severity": "warn" + }, + { + "name": "amount_non_negative", + "check": "min", + "column": "amount", + "condition": {"geq_to": 0} + }, + { + "name": "region_cardinality_reasonable", + "check": "distinct_count", + "column": "region", + "condition": {"leq_to": 20} + }, + { + "name": "table_not_empty", + "check": "row_count", + "condition": {"greater_than": 0} + }, + { + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE created_at > CURRENT_TIMESTAMP", + "condition": {"equal_to": 0} + } + ] +} +``` + +## Reference schema + +`references/ruleset.schema.json` in this skill's directory is the pydantic-generated JSON +Schema for this exact shape (`RuleSet.model_json_schema()`), useful for validating output +structurally. It does not enumerate valid `check` values (that field is a plain string) -- +this document is the source of truth for which check names exist. + +## How this is consumed + +The generated JSON is typically produced by an LLM task (e.g. `@task.llm(output_type=RuleSet, ...)` +from `common.ai`) and passed straight to `DQCheckOperator(ruleset=...)`, `@task.dq_check(ruleset=...)`, +or `asset_quality(ruleset=...)` -- all three accept a `RuleSet`, its dict form, or a path to a +YAML file written in this same shape. Invalid output raises a `pydantic.ValidationError` +describing exactly which field or value was wrong. diff --git a/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json new file mode 100644 index 0000000000000..a8f4387981d6d --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json @@ -0,0 +1,241 @@ +{ + "$defs": { + "Condition": { + "additionalProperties": false, + "description": "Pass/fail condition evaluated against a rule's observed value.\n\nUses the same grammar as the ``common.sql`` check operators: ``equal_to``,\n``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage\n``tolerance`` that widens ``equal_to`` into a range.", + "properties": { + "equal_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Equal To" + }, + "greater_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Greater Than" + }, + "less_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Less Than" + }, + "geq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Geq To" + }, + "leq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Leq To" + }, + "tolerance": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tolerance" + } + }, + "title": "Condition", + "type": "object" + }, + "DQRule": { + "additionalProperties": false, + "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "check": { + "description": "One of the built-in checks, or custom_sql.", + "title": "Check", + "type": "string" + }, + "condition": { + "anyOf": [ + { + "$ref": "#/$defs/Condition" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Condition" + }, + "column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Column" + }, + "sql": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sql" + }, + "severity": { + "$ref": "#/$defs/Severity", + "default": "error" + }, + "partition_clause": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Partition Clause" + }, + "previous_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Previous Name" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Human-readable description shown in results and the UI. When omitted, the provider generates a short default description from the rule and condition.", + "title": "Description" + }, + "dimension": { + "anyOf": [ + { + "$ref": "#/$defs/Dimension" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data quality dimension recorded for this rule's results. Defaults to the check's catalog dimension (validity for custom_sql) when not set explicitly." + } + }, + "required": [ + "name", + "check" + ], + "title": "DQRule", + "type": "object" + }, + "Dimension": { + "description": "Data quality dimension assigned to a rule result.", + "enum": [ + "completeness", + "uniqueness", + "validity", + "freshness", + "volume", + "consistency" + ], + "title": "Dimension", + "type": "string" + }, + "Severity": { + "description": "Severity used to decide whether a failing rule fails the task.", + "enum": [ + "warn", + "error" + ], + "title": "Severity", + "type": "string" + } + }, + "additionalProperties": false, + "description": "A named collection of rules, typically attached to one table or asset.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "rules": { + "default": [], + "items": { + "$ref": "#/$defs/DQRule" + }, + "title": "Rules", + "type": "array" + } + }, + "required": [ + "name" + ], + "title": "RuleSet", + "type": "object" +} diff --git a/providers/dq/src/airflow/providers/dq/version_compat.py b/providers/dq/src/airflow/providers/dq/version_compat.py new file mode 100644 index 0000000000000..b47842705282b --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/version_compat.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY +# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS +# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT +# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE +# +from __future__ import annotations + + +def get_base_airflow_version_tuple() -> tuple[int, int, int]: + from packaging.version import Version + + from airflow import __version__ + + airflow_version = Version(__version__) + return airflow_version.major, airflow_version.minor, airflow_version.micro + + +AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0) + +__all__ = ["AIRFLOW_V_3_1_PLUS"] diff --git a/providers/dq/tests/conftest.py b/providers/dq/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/dq/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/dq/tests/system/__init__.py b/providers/dq/tests/system/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/tests/system/dq/__init__.py b/providers/dq/tests/system/dq/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/system/dq/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/system/dq/example_dq_check.py b/providers/dq/tests/system/dq/example_dq_check.py new file mode 100644 index 0000000000000..62322478145b4 --- /dev/null +++ b/providers/dq/tests/system/dq/example_dq_check.py @@ -0,0 +1,354 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG for the Data Quality provider's ``DQCheckOperator``. + +Runs against the ``sqlite_default`` connection (Airflow's default local backend, no extra +infrastructure required). Results are persisted through the ``[dq] results_path`` config +option, seeded here via ``AIRFLOW__DQ__RESULTS_PATH`` so the history can be inspected +afterwards under ``/tmp/airflow_dq_example/results`` -- a real deployment would instead set +``results_path`` in ``airflow.cfg`` (or its env var) once, for every check to share. + +Trigger the Dag repeatedly with different ``dq_scenario`` values to create useful +persisted data quality history: + +- ``pass`` inserts clean data. +- ``negative`` inserts a negative amount, failing ``amount_non_negative``. +- ``duplicate_order`` inserts a duplicate ``order_id``, failing uniqueness. +- ``nulls`` inserts null ``order_id``/``customer_id`` values and a high discount null ratio. +- ``outlier`` inserts amounts above the allowed maximum. +- ``small_table`` inserts too few rows for volume checks. +- ``zero_quantity`` inserts an invalid quantity minimum. +- ``empty`` leaves the table empty. +- ``mixed`` combines several bad values to show multiple failures in one run. + +Demonstrates +three equivalent ways to run the same ruleset: + +- Plain ``DQCheckOperator`` with an explicit ``table``/``conn_id``. +- ``asset_quality()`` attaching the ruleset to an ``Asset``, then ``DQCheckOperator(asset=...)`` + picking up ``table``/``conn_id``/``ruleset`` from it and adding the asset to the task's + outlets automatically. +- A second asset-producing ``DQCheckOperator`` with stricter rules, so the Browse > Data Quality + page shows multiple producers for the same asset with both pass and fail outcomes. +- The ``@task.dq_check`` TaskFlow decorator. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.assets import asset_quality +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.sdk import DAG, Asset, task + +DAG_ID = "example_dq_check" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_example_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +# DQCheckOperator has no per-operator results-store override; every check in a deployment +# shares the one store configured under [dq] results_path. setdefault() so a real deployment's +# own config is never overridden. +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_ruleset] +orders_ruleset = RuleSet( + name="orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="customer_id_not_null", + check="null_count", + column="customer_id", + condition={"equal_to": 0}, + severity=Severity.WARN, + ), + DQRule( + name="discount_null_ratio", + check="null_ratio", + column="discount", + condition={"leq_to": 0.25}, + severity=Severity.WARN, + ), + DQRule( + name="region_distinct_count", + check="distinct_count", + column="region", + condition={"geq_to": 2}, + severity=Severity.WARN, + ), + DQRule( + name="amount_min_ge_zero", + check="min", + column="amount", + condition={"geq_to": 0}, + ), + DQRule( + name="amount_max_le_100", + check="max", + column="amount", + condition={"leq_to": 100}, + ), + DQRule( + name="amount_mean_between_5_and_60", + check="mean", + column="amount", + condition={"geq_to": 5, "leq_to": 60}, + ), + DQRule( + name="quantity_min_ge_one", + check="min", + column="quantity", + condition={"geq_to": 1}, + ), + DQRule( + name="quantity_max_le_ten", + check="max", + column="quantity", + condition={"leq_to": 10}, + ), + DQRule( + name="amount_non_negative", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string placeholder. + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition={"equal_to": 0}, + ), + DQRule( + name="high_value_order_count", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount > 100", + condition={"equal_to": 0}, + severity=Severity.WARN, + ), + DQRule( + name="row_count_present", + check="row_count", + condition={"greater_than": 0}, + severity=Severity.WARN, + ), + DQRule( + name="row_count_at_least_three", + check="row_count", + condition={"geq_to": 3}, + ), + ), +) +# [END howto_operator_dq_check_ruleset] + +# [START howto_operator_dq_check_asset] +orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_operator_dq_check_asset] + +strict_orders_ruleset = RuleSet( + name="orders_strict_quality", + rules=( + DQRule( + name="strict_order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="strict_amount_max_le_20", + check="max", + column="amount", + condition={"leq_to": 20}, + ), + DQRule( + name="strict_row_count_equal_two", + check="row_count", + condition={"equal_to": 2}, + ), + DQRule( + name="strict_region_distinct_count", + check="distinct_count", + column="region", + condition={"geq_to": 4}, + severity=Severity.WARN, + ), + ), +) + +strict_orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=strict_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) + + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + discount REAL, + quantity INTEGER, + region TEXT + ); + """, + ], + ) + + clear_table = SQLExecuteQueryOperator( + task_id="clear_table", + conn_id=CONN_ID, + sql=f"DELETE FROM {TABLE_NAME};", + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + {{% set scenario = dag_run.conf.get("dq_scenario", "pass") if dag_run else "pass" %}} + {{% if scenario == "negative" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, -25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "duplicate_order" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (1, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "nulls" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 1, 'US'), + (NULL, NULL, 25.5, NULL, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "outlier" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 120.0, 0.10, 2, 'EU'), + (3, 103, 150.0, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "small_table" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'); + {{% elif scenario == "zero_quantity" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 0, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "mixed" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 0, 'US'), + (1, NULL, -25.5, NULL, 2, 'US'), + (NULL, 103, 150.0, NULL, 11, 'US'), + (4, 104, 55.0, 0.20, 4, 'US'); + {{% elif scenario == "empty" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) + SELECT NULL, NULL, NULL, NULL, NULL, NULL WHERE 0; + {{% else %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% endif %}} + """, + ) + + # [START howto_operator_dq_check] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + # [END howto_operator_dq_check] + + check_orders_via_asset = DQCheckOperator( + task_id="check_orders_via_asset", + asset=orders_asset, + fail_on="never", + ) + + check_orders_strict_via_asset = DQCheckOperator( + task_id="check_orders_strict_via_asset", + asset=strict_orders_asset, + fail_on="never", + ) + + # [START howto_decorator_dq_check] + @task.dq_check( + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + def check_orders_decorated(): + return None + + # [END howto_decorator_dq_check] + + ( + create_table + >> clear_table + >> insert_orders + >> [check_orders, check_orders_via_asset, check_orders_strict_via_asset, check_orders_decorated()] + ) + + from tests_common.test_utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/dq/tests/unit/__init__.py b/providers/dq/tests/unit/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/tests/unit/dq/__init__.py b/providers/dq/tests/unit/dq/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/backends/__init__.py b/providers/dq/tests/unit/dq/backends/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/backends/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/backends/test_object_storage.py b/providers/dq/tests/unit/dq/backends/test_object_storage.py new file mode 100644 index 0000000000000..bc6ac2d4b3b28 --- /dev/null +++ b/providers/dq/tests/unit/dq/backends/test_object_storage.py @@ -0,0 +1,357 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from airflow.providers.dq.backends import get_backend_from_config +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.results import DQRun, RuleResult +from airflow.providers.dq.rules import Severity + +from tests_common.test_utils.config import conf_vars + + +def make_run(run_uid: str = "abc123", started_at: str = "2026-07-04T06:00:00+00:00") -> DQRun: + return DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid=run_uid, + ruleset_name="orders", + table_ref="analytics.orders", + asset_names=("dq_example_orders",), + started_at=started_at, + finished_at="2026-07-04T06:00:03+00:00", + ) + + +def make_result(rule_uid: str = "rule-1", status: str = "pass") -> RuleResult: + return RuleResult( + rule_uid=rule_uid, + rule_name="ids_not_null", + status=status, + observed_value=0, + condition={"equal_to": 0}, + dimension="completeness", + severity=Severity.ERROR, + duration_ms=12.5, + ) + + +class TestObjectStorageResultsBackend: + @pytest.fixture + def backend(self, tmp_path): + return ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + def test_write_run_creates_run_file_with_run_and_results(self, backend): + backend.write_run(make_run(), [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + assert record["run"]["run_uid"] == "abc123" + assert record["run"]["dag_id"] == "orders_pipeline" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["results"][0]["status"] == "pass" + + def test_write_run_stores_keyed_json_payload(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = ( + backend.root + / "runs" + / "by_task" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "date=2026-07-04" + / "abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["run_uid"] == "abc123" + assert payload["results"][0]["rule_uid"] == "rule-1" + assert payload["summary"]["passed"] == 1 + + def test_write_run_stores_task_rule_index(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = ( + backend.root + / "rules" + / "by_task_rule" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "rule_uid=rule-1" + / "2026-07-04T06_00_00_00_00__abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["dag_id"] == "orders_pipeline" + assert payload["run"]["task_id"] == "dq" + assert payload["result"]["rule_uid"] == "rule-1" + + def test_rule_history_is_newest_first(self, backend): + backend.write_run( + make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + [make_result(status="pass")], + ) + backend.write_run( + make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + [make_result(status="fail")], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1") + + assert [record["status"] for record in result["items"]] == ["fail", "pass"] + assert result["items"][0]["run"]["run_uid"] == "run2" + assert result["next_cursor"] is None + + def test_rule_history_includes_task_instance_route_context(self, backend): + backend.write_run(make_run(), [make_result()]) + + history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + + assert history[0]["run"]["dag_id"] == "orders_pipeline" + assert history[0]["run"]["task_id"] == "dq" + assert history[0]["run"]["run_id"] == "scheduled__2026-07-04" + assert history[0]["run"]["map_index"] == -1 + + def test_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_concurrent_style_writes_do_not_collide(self, backend): + backend.write_run(make_run(run_uid="run1"), [make_result()]) + backend.write_run(make_run(run_uid="run2"), [make_result()]) + + runs = backend.read_task_runs("orders_pipeline", "dq")["items"] + assert {run["run"]["run_uid"] for run in runs} == {"run1", "run2"} + + def test_read_by_task_instance_matches_run(self, backend): + backend.write_run(make_run(), [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + + assert record["run"]["run_uid"] == "abc123" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_unknown_raises(self, backend): + with pytest.raises(FileNotFoundError): + backend.read_by_task_instance("orders_pipeline", "dq", "no_such_run") + + def test_read_by_task_instance_last_write_wins_across_retries(self, backend): + first = make_run(run_uid="try1") + backend.write_run(first, [make_result(status="fail")]) + second = make_run(run_uid="try2") + backend.write_run(second, [make_result(status="pass")]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + + assert record["run"]["run_uid"] == "try2" + assert record["results"][0]["status"] == "pass" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_sanitizes_run_id(self, backend): + run_id = "manual__2026-07-04T06:00:00+00:00" + run = DQRun(dag_id="orders_pipeline", task_id="dq", run_id=run_id, run_uid="abc123") + backend.write_run(run, [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", run_id) + + assert record["run"]["run_id"] == run_id + + def test_read_task_runs_returns_newest_first_with_summaries(self, backend): + backend.write_run( + make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + [make_result(status="pass")], + ) + backend.write_run( + make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + [make_result(status="fail")], + ) + + result = backend.read_task_runs("orders_pipeline", "dq") + runs = result["items"] + + assert [record["run"]["run_uid"] for record in runs] == ["run2", "run1"] + assert runs[0]["summary"]["failed"] == 1 + assert runs[1]["summary"]["passed"] == 1 + assert result["next_cursor"] is None + + def test_read_task_runs_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=2) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_runs_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + second_page = backend.read_task_runs( + "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + second_page = backend.read_task_runs( + "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monkeypatch): + for day in range(1, 5): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + read_paths: list[Any] = [] + original_read_json = backend._read_json + monkeypatch.setattr( + backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=1) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run4"] + assert result["next_cursor"] == "2026-07-04T06:00:00+00:00|run4" + # Determining next_cursor reads one entry past the limit (date=2026-07-03's run3), but + # no further — date=2026-07-02/01 must not be read. + assert len(read_paths) == 2 + + def test_read_task_rule_history_filters_to_task(self, backend): + backend.write_run(make_run(run_uid="run1"), [make_result()]) + backend.write_run( + DQRun( + dag_id="other_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid="run2", + started_at="2026-07-04T07:00:00+00:00", + ), + [make_result()], + ) + + history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + + assert len(history) == 1 + assert history[0]["run"]["dag_id"] == "orders_pipeline" + + def test_read_task_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run3", "run2"] + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_rule_history_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + +class TestGetBackendFromConfig: + @conf_vars({("dq", "results_path"): None}) + def test_no_results_path_returns_none(self): + assert get_backend_from_config() is None + + def test_results_path_builds_object_storage_backend(self, tmp_path): + with conf_vars({("dq", "results_path"): f"file://{tmp_path}"}): + backend = get_backend_from_config() + assert isinstance(backend, ObjectStorageResultsBackend) diff --git a/providers/dq/tests/unit/dq/decorators/__init__.py b/providers/dq/tests/unit/dq/decorators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/decorators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/decorators/test_dq_check.py b/providers/dq/tests/unit/dq/decorators/test_dq_check.py new file mode 100644 index 0000000000000..cbe88a1fb69f6 --- /dev/null +++ b/providers/dq/tests/unit/dq/decorators/test_dq_check.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.decorators.dq_check import _DQCheckDecoratedOperator +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +RULESET = RuleSet(name="orders", rules=(NULLS,)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch("airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend): + yield backend + + +def make_decorated_operator(records, python_callable, **kwargs): + kwargs.setdefault("task_id", "dq") + if not kwargs.pop("omit_ruleset", False): + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = _DQCheckDecoratedOperator(conn_id="warehouse", python_callable=python_callable, **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +class TestDQCheckDecoratedOperator: + def test_custom_operator_name(self): + assert _DQCheckDecoratedOperator.custom_operator_name == "@task.dq_check" + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_none_return_runs_check_as_declared(self, mock_get_db_hook): + operator, hook = make_decorated_operator(records=[(NULLS.rule_uid, 0)], python_callable=lambda: None) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert operator.table == "orders" + assert summary["passed"] == 1 + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_ruleset_can_be_provided_only_at_runtime(self, mock_get_db_hook): + other_rule = DQRule(name="row_count_ok", check="row_count", condition={"greater_than": 0}) + other_ruleset = RuleSet(name="dynamic", rules=(other_rule,)) + + operator, hook = make_decorated_operator( + records=[(other_rule.rule_uid, 10)], + python_callable=lambda: other_ruleset, + omit_ruleset=True, + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 1 + + def test_missing_runtime_ruleset_raises_value_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: None, omit_ruleset=True) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(ValueError, match="ruleset is required"): + operator.execute(make_context()) + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_op_kwargs_are_passed_to_callable(self, mock_get_db_hook): + seen = {} + + def resolve_target(suffix): + seen["suffix"] = suffix + return None + + operator, hook = make_decorated_operator( + records=[(NULLS.rule_uid, 0)], + python_callable=resolve_target, + op_kwargs={"suffix": "eu"}, + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + assert seen["suffix"] == "eu" + + def test_invalid_return_raises_type_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: 42) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(TypeError, match="must return a RuleSet"): + operator.execute(make_context()) diff --git a/providers/dq/tests/unit/dq/engines/__init__.py b/providers/dq/tests/unit/dq/engines/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/engines/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/engines/test_sql.py b/providers/dq/tests/unit/dq/engines/test_sql.py new file mode 100644 index 0000000000000..9161275c3849a --- /dev/null +++ b/providers/dq/tests/unit/dq/engines/test_sql.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.dq.engines.sql import SQLDQEngine +from airflow.providers.dq.rules import DQRule, RuleSet + + +@pytest.fixture +def hook(): + return mock.create_autospec(DbApiHook, instance=True) + + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +VOLUME = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) +CUSTOM = DQRule( + name="negatives", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition={"equal_to": 0}, +) + + +class TestBuildBatchSql: + def test_batch_sql_unions_one_select_per_rule(self, hook): + engine = SQLDQEngine(hook) + sql = engine.build_batch_sql([NULLS, VOLUME], "orders", None) + assert sql.count("UNION ALL") == 1 + assert f"'{NULLS.rule_uid}' AS rule_uid" in sql + assert "SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END)" in sql + assert "COUNT(*)" in sql + assert "WHERE" not in sql + + def test_partition_clauses_are_anded(self, hook): + rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition={"equal_to": 0}, + partition_clause="region = 'EU'", + ) + sql = SQLDQEngine(hook).build_batch_sql([rule], "orders", "ds = '2026-07-04'") + assert "WHERE ds = '2026-07-04' AND region = 'EU'" in sql + + +class TestMeasure: + def test_builtin_rules_measured_from_single_query(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 42)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + hook.get_records.assert_called_once() + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].observed_value == 0 + assert by_name["volume"].observed_value == 42 + assert by_name["nulls"].sql == SQLDQEngine(hook).build_rule_sql(NULLS, "orders") + assert by_name["volume"].sql == SQLDQEngine(hook).build_rule_sql(VOLUME, "orders") + assert all(obs.error_message is None for obs in observations) + + def test_query_failure_yields_error_observations(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + assert len(observations) == 2 + assert all(obs.error_message == "connection refused" for obs in observations) + assert all(obs.observed_value is None for obs in observations) + assert all(obs.sql for obs in observations) + + def test_missing_rule_in_result_is_an_error(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].error_message is None + assert by_name["volume"].error_message == "No result returned for rule" + + def test_custom_sql_rule_runs_individually_with_table_substituted(self, hook): + hook.get_first.return_value = (3,) + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + hook.get_first.assert_called_once_with("SELECT COUNT(*) FROM orders WHERE amount < 0") + hook.get_records.assert_not_called() + assert observations[0].observed_value == 3 + assert observations[0].sql == "SELECT COUNT(*) FROM orders WHERE amount < 0" + + def test_custom_sql_no_rows_is_an_error(self, hook): + hook.get_first.return_value = None + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + assert observations[0].error_message == "Query returned no rows" diff --git a/providers/dq/tests/unit/dq/operators/__init__.py b/providers/dq/tests/unit/dq/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/operators/test_dq_check.py b/providers/dq/tests/unit/dq/operators/test_dq_check.py new file mode 100644 index 0000000000000..9e74a8dc94642 --- /dev/null +++ b/providers/dq/tests/unit/dq/operators/test_dq_check.py @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.dq.assets import asset_quality +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.exceptions import DQCheckFailedError +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.sdk import Asset +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +VOLUME_WARN = DQRule( + name="volume", check="row_count", condition={"greater_than": 100}, severity=Severity.WARN +) +RULESET = RuleSet(name="orders", rules=(NULLS, VOLUME_WARN)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +def make_operator(records, **kwargs): + kwargs.setdefault("task_id", "dq") + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = DQCheckOperator(conn_id="warehouse", **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch("airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend): + yield backend + + +class TestDQCheckOperatorConstruction: + def test_requires_ruleset(self): + with pytest.raises(ValueError, match="ruleset is required"): + DQCheckOperator(task_id="dq", table="orders", conn_id="c") + + def test_requires_table(self): + with pytest.raises(ValueError, match="table is required"): + DQCheckOperator(task_id="dq", ruleset=RULESET, conn_id="c") + + def test_rejects_bad_fail_on(self): + with pytest.raises(ValueError, match="fail_on"): + DQCheckOperator(task_id="dq", ruleset=RULESET, table="orders", conn_id="c", fail_on="maybe") + + def test_asset_supplies_defaults_and_outlet(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset) + assert operator.table == "analytics.orders" + assert operator.conn_id == "warehouse" + assert asset in operator.outlets + + def test_explicit_arguments_beat_asset_config(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset, table="other_table", conn_id="other_conn") + assert operator.table == "other_table" + assert operator.conn_id == "other_conn" + + +class TestDQCheckOperatorExecute: + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_all_pass_returns_summary_and_persists(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + assert summary["failed"] == 0 + assert summary["score"] == 1.0 + history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + assert len(history) == 1 + assert history[0]["status"] == "pass" + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_error_severity_failure_fails_task(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="nulls"): + operator.execute(make_context()) + + # The failing result is persisted even though the task fails. + assert ( + results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"][0][ + "status" + ] + == "fail" + ) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_warn_severity_failure_does_not_fail_task_by_default(self, mock_get_db_hook): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["warned"] == 1 + assert summary["warned_rules"] == ["volume"] + assert summary["score"] == pytest.approx(0.875) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_rule_description_is_persisted(self, mock_get_db_hook, results_backend): + described_rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition={"equal_to": 0}, + description="Order IDs must always be present.", + ) + operator, hook = make_operator( + records=[(described_rule.rule_uid, 0)], + ruleset=RuleSet(name="orders", rules=(described_rule,)), + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + history = results_backend.read_task_rule_history("orders_pipeline", "dq", described_rule.rule_uid)[ + "items" + ] + assert history[0]["description"] == "Order IDs must always be present." + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_warn_fails_task_on_warn_failure(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)], fail_on="warn" + ) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="volume"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_never_records_failures_without_raising(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 5)], fail_on="never" + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["failed"] == 1 + assert summary["warned"] == 1 + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_execution_error_always_fails_task(self, mock_get_db_hook): + operator, hook = make_operator(records=None, fail_on="never") + hook.get_records.side_effect = RuntimeError("boom") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_non_numeric_observed_value_is_persisted_as_error(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, "not-a-number")], fail_on="never") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + assert history[0]["status"] == "error" + assert "Could not evaluate observed value" in history[0]["error_message"] + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_backend_failure_does_not_fail_the_check(self, mock_get_db_hook): + backend = mock.create_autospec(ObjectStorageResultsBackend, instance=True) + backend.write_run.side_effect = OSError("bucket unreachable") + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with mock.patch( + "airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend + ): + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + backend.write_run.assert_called_once() + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_summary_attached_to_outlet_events(self, mock_get_db_hook): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="orders", + ) + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)], asset=asset + ) + mock_get_db_hook.return_value = hook + context = make_context() + + summary = operator.execute(context) + + outlet_events = context["outlet_events"] + outlet_events.__getitem__.assert_called_with(asset) + extra = outlet_events.__getitem__.return_value.extra + extra.__setitem__.assert_called_once_with("airflow.dq.result", summary) diff --git a/providers/dq/tests/unit/dq/rules/__init__.py b/providers/dq/tests/unit/dq/rules/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/rules/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/rules/test_checks.py b/providers/dq/tests/unit/dq/rules/test_checks.py new file mode 100644 index 0000000000000..8221a9f040da7 --- /dev/null +++ b/providers/dq/tests/unit/dq/rules/test_checks.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.rules import CHECK_SPECS + + +class TestCheckSpecs: + def test_every_column_check_expression_renders(self): + for name, spec in CHECK_SPECS.items(): + if spec.requires_column: + assert "{column}" in spec.expression, name + + def test_row_count_is_the_only_table_level_check(self): + table_level = {name for name, spec in CHECK_SPECS.items() if not spec.requires_column} + assert table_level == {"row_count"} diff --git a/providers/dq/tests/unit/dq/rules/test_rule.py b/providers/dq/tests/unit/dq/rules/test_rule.py new file mode 100644 index 0000000000000..4c5cd46d38784 --- /dev/null +++ b/providers/dq/tests/unit/dq/rules/test_rule.py @@ -0,0 +1,264 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import airflow.providers.dq +from airflow.providers.dq.rules import Condition, DQRule, RuleSet, Severity, describe_rule + + +def test_skill_reference_schema_matches_live_model(): + """ + The dq-rule-authoring skill ships a generated RuleSet.model_json_schema() snapshot for + agents to consult. Guard against it silently drifting out of sync with the real model -- + regenerate with ``json.dumps(RuleSet.model_json_schema(), indent=2)`` (plus a trailing + newline) if this fails after an intentional schema change. + """ + schema_path = ( + Path(airflow.providers.dq.__file__).parent + / "skills" + / "dq-rule-authoring" + / "references" + / "ruleset.schema.json" + ) + checked_in = json.loads(schema_path.read_text()) + assert checked_in == RuleSet.model_json_schema() + + +class TestCondition: + @pytest.mark.parametrize( + ("condition", "observed", "expected"), + [ + ({"equal_to": 0}, 0, True), + ({"equal_to": 0}, 1, False), + ({"equal_to": 100, "tolerance": 0.1}, 105, True), + ({"equal_to": 100, "tolerance": 0.1}, 111, False), + ({"greater_than": 5}, 6, True), + ({"greater_than": 5}, 5, False), + ({"less_than": 5}, 4, True), + ({"geq_to": 5}, 5, True), + ({"leq_to": 5}, 6, False), + ({"geq_to": 0, "leq_to": 10}, 5, True), + ({"geq_to": 0, "leq_to": 10}, 11, False), + ({"equal_to": 0}, None, False), + ], + ) + def test_evaluate(self, condition, observed, expected): + assert Condition.from_dict(condition).evaluate(observed) is expected + + @pytest.mark.parametrize( + "condition", + [ + {}, + {"tolerance": 0.1}, + {"equal_to": 1, "greater_than": 0}, + {"greater_than": 0, "tolerance": 0.1}, + {"nonsense": 1}, + ], + ) + def test_invalid_conditions_rejected(self, condition): + with pytest.raises(ValidationError): + Condition.from_dict(condition) + + def test_to_dict_round_trip(self): + condition = Condition.from_dict({"geq_to": 1, "leq_to": 2}) + assert Condition.from_dict(condition.to_dict()) == condition + + +class TestDQRule: + def test_condition_dict_is_coerced(self): + rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + assert isinstance(rule.condition, Condition) + + def test_rule_uid_is_stable_across_severity(self): + base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + tweaked = DQRule( + name="r", + check="null_count", + column="c", + condition={"equal_to": 0}, + severity=Severity.WARN, + ) + assert base.rule_uid == tweaked.rule_uid + + def test_rule_uid_is_stable_across_description(self): + base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + described = DQRule( + name="r", + check="null_count", + column="c", + condition={"equal_to": 0}, + description="Column c must be populated.", + ) + assert base.rule_uid == described.rule_uid + + def test_rule_uid_changes_with_condition(self): + one = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + two = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 1}) + assert one.rule_uid != two.rule_uid + + def test_previous_name_keeps_uid(self): + old = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition={"equal_to": 0}, + previous_name="old_name", + ) + assert renamed.rule_uid == old.rule_uid + + @pytest.mark.parametrize( + "kwargs", + [ + {"name": "", "check": "row_count", "condition": {"equal_to": 0}}, + {"name": "r", "check": "no_such_check", "condition": {"equal_to": 0}}, + {"name": "r", "check": "null_count", "condition": {"equal_to": 0}}, # missing column + {"name": "r", "check": "custom_sql", "condition": {"equal_to": 0}}, # missing sql + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "sql": "SELECT 1"}, + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "severity": "fatal"}, + ], + ) + def test_invalid_rules_rejected(self, kwargs): + with pytest.raises(ValidationError): + DQRule(**kwargs) + + def test_row_count_needs_no_column(self): + rule = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) + assert rule.column is None + + @pytest.mark.parametrize( + ("check", "column", "expected_dimension"), + [ + ("null_count", "c", "completeness"), + ("null_ratio", "c", "completeness"), + ("distinct_count", "c", "uniqueness"), + ("unique_violations", "c", "uniqueness"), + ("min", "c", "validity"), + ("max", "c", "validity"), + ("mean", "c", "validity"), + ("row_count", None, "volume"), + ], + ) + def test_default_dimension_comes_from_check_catalog(self, check, column, expected_dimension): + rule = DQRule(name="r", check=check, column=column, condition={"equal_to": 0}) + assert rule.dimension.value == expected_dimension + + def test_custom_sql_dimension_can_be_overridden(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition={"equal_to": 0}, + dimension="freshness", + ) + assert rule.dimension.value == "freshness" + + def test_invalid_dimension_rejected(self): + with pytest.raises(ValidationError): + DQRule(name="r", check="row_count", condition={"greater_than": 0}, dimension="not_a_dimension") + + def test_explicit_dimension_matching_default_is_omitted_from_to_dict(self): + rule = DQRule( + name="r", check="null_count", column="c", condition={"equal_to": 0}, dimension="completeness" + ) + assert "dimension" not in rule.to_dict() + + def test_dimension_override_round_trips(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition={"equal_to": 0}, + dimension="freshness", + ) + assert rule.to_dict()["dimension"] == "freshness" + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_missing_condition_without_catalog_default_raises(self): + with pytest.raises(ValidationError, match="condition is required"): + DQRule(name="r", check="null_count", column="c") + + def test_to_dict_round_trip(self): + rule = DQRule( + name="r", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE x < 0", + condition={"equal_to": 0}, + severity=Severity.WARN, + partition_clause="ds = '2026-07-04'", + description="No rows should have negative x.", + ) + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_description_is_serialized(self): + rule = DQRule( + name="r", + check="row_count", + condition={"greater_than": 0}, + description="Orders table should not be empty.", + ) + assert rule.to_dict()["description"] == "Orders table should not be empty." + + def test_default_description_uses_column_when_available(self): + rule = DQRule(name="amount_min", check="min", column="amount", condition={"geq_to": 0}) + assert describe_rule(rule) == "amount should be greater than or equal to 0.0" + + +class TestRuleSet: + def test_duplicate_rule_names_rejected(self): + rule = DQRule(name="r", check="row_count", condition={"greater_than": 0}) + with pytest.raises(ValidationError, match="Duplicate rule names"): + RuleSet(name="s", rules=(rule, rule)) + + def test_from_dict_round_trip(self): + ruleset = RuleSet( + name="orders", + rules=( + DQRule(name="volume", check="row_count", condition={"greater_than": 0}), + DQRule(name="ids", check="null_count", column="id", condition={"equal_to": 0}), + ), + ) + assert RuleSet.from_dict(ruleset.to_dict()) == ruleset + + def test_from_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: ids_not_null + check: null_count + column: id + condition: + equal_to: 0 + - name: volume + check: row_count + condition: + greater_than: 100 + severity: warn + """ + ) + ruleset = RuleSet.from_file(str(ruleset_file)) + assert ruleset.name == "orders" + assert [rule.name for rule in ruleset.rules] == ["ids_not_null", "volume"] + assert ruleset.rules[1].severity.value == "warn" diff --git a/providers/dq/tests/unit/dq/test_assets.py b/providers/dq/tests/unit/dq/test_assets.py new file mode 100644 index 0000000000000..9079b0ce567d9 --- /dev/null +++ b/providers/dq/tests/unit/dq/test_assets.py @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.dq.assets import ( + DQ_EXTRA_KEY, + DQ_RESULT_EXTRA_KEY, + _quality_score_passes, + asset_quality, + get_asset_quality_config, + get_asset_ruleset, + require_quality, +) +from airflow.providers.dq.exceptions import DQRuleValidationError +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk import Asset +from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult +from airflow.sdk.execution_time.context import TriggeringAssetEventsAccessor + +RULESET = RuleSet( + name="orders", + rules=(DQRule(name="volume", check="row_count", condition={"greater_than": 0}),), +) + +ORDERS = Asset("orders") + + +def make_triggering_events(*, extra: dict, asset: Asset = ORDERS) -> TriggeringAssetEventsAccessor: + event = { + "asset": {"name": asset.name, "uri": asset.uri, "extra": {}}, + "extra": extra, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "manual__2026-07-04", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-04T06:00:00Z", + } + return TriggeringAssetEventsAccessor.build([AssetEventDagRunReferenceResult.model_validate(event)]) + + +class TestAssetQuality: + def test_config_stored_under_dq_extra_key(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + config = asset.extra[DQ_EXTRA_KEY] + assert config["conn_id"] == "warehouse" + assert config["table"] == "analytics.orders" + assert config["ruleset"] == RULESET.to_dict() + + def test_config_is_json_serializable(self): + import json + + asset = asset_quality(Asset("orders"), ruleset=RULESET) + json.dumps(asset.extra) + + def test_ruleset_round_trips_through_asset(self): + asset = asset_quality(Asset("orders"), ruleset=RULESET) + assert get_asset_ruleset(asset) == RULESET + + def test_ruleset_from_yaml_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: volume + check: row_count + condition: + greater_than: 0 + """ + ) + asset = asset_quality(Asset("orders"), ruleset=str(ruleset_file)) + assert get_asset_ruleset(asset).rules[0].name == "volume" + + def test_get_config_returns_none_without_quality(self): + assert get_asset_quality_config(Asset("plain")) is None + + def test_get_ruleset_raises_without_quality(self): + with pytest.raises(DQRuleValidationError, match="no data quality config"): + get_asset_ruleset(Asset("plain")) + + +class TestQualityScorePasses: + """ + Covers the pure decision logic behind ``require_quality()``. + + Kept independent of ``@task.short_circuit`` / Dag wiring, which is exercised by the + standard provider's own tests — this only needs to prove "given this triggering event, + does the gate pass or fail". + """ + + def test_passes_when_score_at_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.95}}) + assert _quality_score_passes(ORDERS, 0.95, events) is True + + def test_fails_when_score_below_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.8}}) + assert _quality_score_passes(ORDERS, 0.95, events) is False + + def test_fails_when_no_triggering_event_for_asset(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 1.0}}, asset=Asset("other")) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_event_has_no_dq_summary(self): + events = make_triggering_events(extra={}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_summary_has_no_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"passed": 3}}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_uses_most_recent_event_when_several_present(self): + first = AssetEventDagRunReferenceResult.model_validate( + { + "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, + "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "run1", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-03T06:00:00Z", + } + ) + second = AssetEventDagRunReferenceResult.model_validate( + { + "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, + "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "run2", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-04T06:00:00Z", + } + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events) is True + + +class TestRequireQuality: + def test_rejects_min_score_below_zero(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=-0.1) + + def test_rejects_min_score_above_one(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=1.1) diff --git a/providers/dq/tests/unit/dq/test_results.py b/providers/dq/tests/unit/dq/test_results.py new file mode 100644 index 0000000000000..76b37ad1b1714 --- /dev/null +++ b/providers/dq/tests/unit/dq/test_results.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.dq.results import DQRun, RuleResult, build_summary, compute_score + + +def make_result(status: str, name: str = "r") -> RuleResult: + return RuleResult(rule_uid=f"uid-{name}", rule_name=name, status=status) + + +class TestComputeScore: + @pytest.mark.parametrize( + ("statuses", "expected"), + [ + ([], None), + (["pass", "pass"], 1.0), + (["pass", "fail"], 0.5), + (["pass", "error"], 0.5), + (["pass", "pass", "pass", "warn"], 0.9375), + (["fail", "fail"], 0.0), + ], + ) + def test_score(self, statuses, expected): + results = [make_result(status, name=f"r{i}") for i, status in enumerate(statuses)] + assert compute_score(results) == expected + + +class TestBuildSummary: + def test_summary_counts_and_rule_names(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", ruleset_name="orders", table_ref="orders") + results = [ + make_result("pass", "a"), + make_result("warn", "b"), + make_result("fail", "c"), + make_result("error", "d"), + ] + + summary = build_summary(run, results) + + assert summary["passed"] == 1 + assert summary["warned"] == 1 + assert summary["failed"] == 1 + assert summary["errored"] == 1 + assert summary["failed_rules"] == ["c", "d"] + assert summary["warned_rules"] == ["b"] + assert summary["run_uid"] == run.run_uid + + +class TestSerialization: + def test_rule_result_round_trip(self): + result = RuleResult( + rule_uid="u", + rule_name="r", + status="fail", + observed_value=3, + condition={"equal_to": 0}, + description="r should equal 0", + duration_ms=1.5, + sql="SELECT 3", + ) + assert RuleResult.from_dict(result.to_dict()) == result + + def test_run_round_trip(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", asset_names=("orders",)) + assert DQRun.from_dict(run.to_dict()) == run diff --git a/pyproject.toml b/pyproject.toml index 87440e9ad9d30..b61a987612779 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -221,6 +221,9 @@ apache-airflow = "airflow.__main__:main" "docker" = [ "apache-airflow-providers-docker>=3.14.1" ] +"dq" = [ + "apache-airflow-providers-dq>=0.1.0" +] "edge3" = [ "apache-airflow-providers-edge3>=1.0.0" ] @@ -454,6 +457,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-dingding>=3.7.0", "apache-airflow-providers-discord>=3.9.0", "apache-airflow-providers-docker>=3.14.1", + "apache-airflow-providers-dq>=0.1.0", "apache-airflow-providers-edge3>=1.0.0", "apache-airflow-providers-elasticsearch>=6.5.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-exasol>=4.6.1", @@ -1172,6 +1176,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/discord/tests", "$MYPY_CONFIG_FILE_DIR/providers/docker/src", "$MYPY_CONFIG_FILE_DIR/providers/docker/tests", + "$MYPY_CONFIG_FILE_DIR/providers/dq/src", + "$MYPY_CONFIG_FILE_DIR/providers/dq/tests", "$MYPY_CONFIG_FILE_DIR/providers/edge3/src", "$MYPY_CONFIG_FILE_DIR/providers/edge3/tests", "$MYPY_CONFIG_FILE_DIR/providers/elasticsearch/src", @@ -1476,6 +1482,7 @@ apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false apache-airflow-providers-docker = false +apache-airflow-providers-dq = false apache-airflow-providers-edge3 = false apache-airflow-providers-elasticsearch = false apache-airflow-providers-exasol = false @@ -1628,6 +1635,7 @@ apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false apache-airflow-providers-docker = false +apache-airflow-providers-dq = false apache-airflow-providers-edge3 = false apache-airflow-providers-elasticsearch = false apache-airflow-providers-exasol = false @@ -1791,6 +1799,7 @@ apache-airflow-providers-dbt-cloud = { workspace = true } apache-airflow-providers-dingding = { workspace = true } apache-airflow-providers-discord = { workspace = true } apache-airflow-providers-docker = { workspace = true } +apache-airflow-providers-dq = { workspace = true } apache-airflow-providers-edge3 = { workspace = true } apache-airflow-providers-elasticsearch = { workspace = true } apache-airflow-providers-exasol = { workspace = true } @@ -1932,6 +1941,7 @@ members = [ "providers/dingding", "providers/discord", "providers/docker", + "providers/dq", "providers/edge3", "providers/elasticsearch", "providers/exasol", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index edb644a179a19..51afe0daa4df7 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -68,6 +68,7 @@ services: - ../../../empty:/opt/airflow/providers/dingding/src - ../../../empty:/opt/airflow/providers/discord/src - ../../../empty:/opt/airflow/providers/docker/src + - ../../../empty:/opt/airflow/providers/dq/src - ../../../empty:/opt/airflow/providers/edge3/src - ../../../empty:/opt/airflow/providers/elasticsearch/src - ../../../empty:/opt/airflow/providers/exasol/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index eb700548346da..e897b9b0ab948 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -81,6 +81,7 @@ services: - ../../../providers/dingding/tests:/opt/airflow/providers/dingding/tests - ../../../providers/discord/tests:/opt/airflow/providers/discord/tests - ../../../providers/docker/tests:/opt/airflow/providers/docker/tests + - ../../../providers/dq/tests:/opt/airflow/providers/dq/tests - ../../../providers/edge3/tests:/opt/airflow/providers/edge3/tests - ../../../providers/elasticsearch/tests:/opt/airflow/providers/elasticsearch/tests - ../../../providers/exasol/tests:/opt/airflow/providers/exasol/tests diff --git a/scripts/ci/prek/generate_dq_ruleset_schema.py b/scripts/ci/prek/generate_dq_ruleset_schema.py new file mode 100755 index 0000000000000..348c8d015b122 --- /dev/null +++ b/scripts/ci/prek/generate_dq_ruleset_schema.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Regenerate the Data Quality RuleSet JSON schema snapshot at +``providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json``. +""" + +from __future__ import annotations + +import subprocess +import sys + +from common_prek_utils import AIRFLOW_ROOT_PATH, console + +DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "dq" +SCHEMA_PATH = DQ_PROVIDER_PATH.joinpath( + "src", + "airflow", + "providers", + "dq", + "skills", + "dq-rule-authoring", + "references", + "ruleset.schema.json", +) + +DUMP_SCHEMA = r""" +import json +import sys + +from airflow.providers.dq.rules import RuleSet + +sys.stdout.write(json.dumps(RuleSet.model_json_schema(), indent=2)) +sys.stdout.write("\n") +""" + + +def dump_schema() -> str: + """Run the schema-dump snippet in the dq provider project and return its stdout.""" + result = subprocess.run( + [ + "uv", + "run", + "--frozen", + "--no-progress", + "--project", + str(DQ_PROVIDER_PATH), + "python", + "-c", + DUMP_SCHEMA, + ], + cwd=DQ_PROVIDER_PATH, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"Schema generation failed: {result.stderr}") + return result.stdout + + +def main() -> int: + try: + new_content = dump_schema() + except Exception as e: + console.print(f"[bold red]ERROR:[/] {e}") + return 1 + + if SCHEMA_PATH.exists(): + old_content = SCHEMA_PATH.read_text() + if old_content == new_content: + return 0 + else: + SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) + + SCHEMA_PATH.write_text(new_content) + rel = SCHEMA_PATH.relative_to(AIRFLOW_ROOT_PATH) + console.print(f"[yellow]Regenerated[/] [cyan]{rel}[/]. Please review the diff and re-stage the file.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/uv.lock b/uv.lock index dc07848533e42..b324a177b64ce 100644 --- a/uv.lock +++ b/uv.lock @@ -64,9 +64,9 @@ apache-airflow-providers-apache-cassandra = false apache-airflow-providers-asana = false apache-airflow-providers-oracle = false apache-airflow-providers-mysql = false +apache-airflow-providers-teradata = false apache-airflow-providers-alibaba = false apache-airflow-providers-microsoft-mssql = false -apache-airflow-providers-teradata = false apache-airflow-providers-jdbc = false apache-airflow-helm-chart = false apache-airflow-providers-anthropic = false @@ -81,6 +81,7 @@ apache-airflow-providers-weaviate = false apache-airflow-providers-akeyless = false apache-airflow-providers-salesforce = false apache-airflow-providers-ssh = false +apache-airflow-providers-dq = false apache-airflow-providers-papermill = false apache-airflow-providers-google = false pydantic-ai-skills = "2026-07-16T00:00:00Z" @@ -215,6 +216,7 @@ members = [ "apache-airflow-providers-dingding", "apache-airflow-providers-discord", "apache-airflow-providers-docker", + "apache-airflow-providers-dq", "apache-airflow-providers-edge3", "apache-airflow-providers-elasticsearch", "apache-airflow-providers-exasol", @@ -330,7 +332,7 @@ name = "adbc-driver-manager" version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/5e/50aab18cb501e42d3aca3cd2cc26c6637094fcaf5b6576e350c444188f1f/adbc_driver_manager-1.11.0.tar.gz", hash = "sha256:c64aaabeb5810109ab3d2961008f1b014e9f2d87b3df4416c2a080a40237af50", size = 233059, upload-time = "2026-04-07T00:17:28.263Z" } wheels = [ @@ -375,8 +377,8 @@ name = "adbc-driver-postgresql" version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "adbc-driver-manager", marker = "python_full_version < '3.13'" }, - { name = "importlib-resources", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, ] sdist = { url = "https://files.pythonhosted.org/packages/68/10/2962c25035887cd03af3b348eac3302493936f45048c220021a802d07f12/adbc_driver_postgresql-1.11.0.tar.gz", hash = "sha256:f5688b8648ac7a86d8b89340231bb3686ac5df56ee95d1ca0b875dad5d52b48a", size = 32328, upload-time = "2026-04-07T00:17:29.232Z" } wheels = [ @@ -392,8 +394,8 @@ name = "adbc-driver-sqlite" version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "adbc-driver-manager", marker = "python_full_version < '3.13'" }, - { name = "importlib-resources", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3a/dd/8a5f4908aa4bdec64dcd672734fa314d692517458ce169591639d0123fe1/adbc_driver_sqlite-1.11.0.tar.gz", hash = "sha256:a4c6b4962610f7cd67cd754c42dd74e18a2c11fabeec9488c5501d73ae62dc62", size = 28885, upload-time = "2026-04-07T00:17:31.325Z" } wheels = [ @@ -456,7 +458,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "caio", marker = "python_full_version < '3.11'" }, + { name = "caio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943, upload-time = "2024-10-08T10:39:35.846Z" } wheels = [ @@ -480,7 +482,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "caio", marker = "python_full_version >= '3.11'" }, + { name = "caio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/41/2fea7e193e061ce54eacc3b7bc0e6a99e4fcff43c78cf0a76dd781ed8334/aiofile-3.11.1.tar.gz", hash = "sha256:1f91912c6643d2a4e49ca4ae3514f0bf3867ce948a36d99a6411b8f4755f4cf9", size = 19342, upload-time = "2026-05-16T08:18:33.538Z" } wheels = [ @@ -964,16 +966,16 @@ vertex = [ [[package]] name = "anyio" -version = "4.14.1" +version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "idna" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] [[package]] @@ -1042,6 +1044,7 @@ all = [ { name = "apache-airflow-providers-dingding" }, { name = "apache-airflow-providers-discord" }, { name = "apache-airflow-providers-docker" }, + { name = "apache-airflow-providers-dq" }, { name = "apache-airflow-providers-edge3" }, { name = "apache-airflow-providers-elasticsearch" }, { name = "apache-airflow-providers-exasol" }, @@ -1249,6 +1252,9 @@ discord = [ docker = [ { name = "apache-airflow-providers-docker" }, ] +dq = [ + { name = "apache-airflow-providers-dq" }, +] edge3 = [ { name = "apache-airflow-providers-edge3" }, ] @@ -1656,6 +1662,8 @@ requires-dist = [ { name = "apache-airflow-providers-discord", marker = "extra == 'discord'", editable = "providers/discord" }, { name = "apache-airflow-providers-docker", marker = "extra == 'all'", editable = "providers/docker" }, { name = "apache-airflow-providers-docker", marker = "extra == 'docker'", editable = "providers/docker" }, + { name = "apache-airflow-providers-dq", marker = "extra == 'all'", editable = "providers/dq" }, + { name = "apache-airflow-providers-dq", marker = "extra == 'dq'", editable = "providers/dq" }, { name = "apache-airflow-providers-edge3", marker = "extra == 'all'", editable = "providers/edge3" }, { name = "apache-airflow-providers-edge3", marker = "extra == 'edge3'", editable = "providers/edge3" }, { name = "apache-airflow-providers-elasticsearch", marker = "extra == 'all'", editable = "providers/elasticsearch" }, @@ -1794,7 +1802,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.28" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-dataquality", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-dataquality", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "dq", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -4813,6 +4821,9 @@ dependencies = [ ] [package.optional-dependencies] +amazon = [ + { name = "apache-airflow-providers-amazon" }, +] avro = [ { name = "fastavro" }, ] @@ -4842,6 +4853,7 @@ standard = [ dev = [ { name = "apache-airflow" }, { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-amazon" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-databricks", extra = ["sqlalchemy"] }, @@ -4860,6 +4872,7 @@ docs = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.14.0,<4" }, { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-amazon", marker = "extra == 'amazon'", editable = "providers/amazon" }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-fab", marker = "extra == 'fab'", editable = "providers/fab" }, @@ -4882,12 +4895,13 @@ requires-dist = [ { name = "pyarrow", marker = "python_full_version >= '3.14'", specifier = ">=22.0.0" }, { name = "requests", specifier = ">=2.32.0,<3" }, ] -provides-extras = ["avro", "azure-identity", "fab", "google", "sdk", "standard", "openlineage", "sqlalchemy"] +provides-extras = ["avro", "amazon", "azure-identity", "fab", "google", "sdk", "standard", "openlineage", "sqlalchemy"] [package.metadata.requires-dev] dev = [ { name = "apache-airflow", editable = "." }, { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-amazon", editable = "providers/amazon" }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-common-sql", extras = ["pandas", "polars"], editable = "providers/common/sql" }, @@ -5110,6 +5124,49 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-dq" +version = "0.1.0" +source = { editable = "providers/dq" } +dependencies = [ + { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "pydantic", specifier = ">=2.11.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-edge3" version = "4.1.0" @@ -10425,8 +10482,8 @@ name = "cassandra-driver" version = "3.30.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "deprecated", marker = "python_full_version < '3.14'" }, - { name = "geomet", marker = "python_full_version < '3.14'" }, + { name = "deprecated" }, + { name = "geomet" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/ed/4e16210e194660f107929ee494f1cf18557252655067d9b39029c241be2d/cassandra_driver-3.30.1.tar.gz", hash = "sha256:0c6a3e1428f7c6a9aa6c944b9c47a37cd2cdbeb5b5a82d42c33afdd56f14e398", size = 289233, upload-time = "2026-07-06T20:14:31.37Z" } wheels = [ @@ -11048,100 +11105,100 @@ wheels = [ [[package]] name = "coverage" -version = "7.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/8b/adeb62ea8951f13c4c7fef2e7a85e1a06b499c8d8237ea589d496029e53f/coverage-7.15.0.tar.gz", hash = "sha256:9ac3fe7a1435986463eaa8ee253ae2f2a268709ba4ae5c7dd1f52a05391ad78f", size = 925362, upload-time = "2026-07-02T13:10:50.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/97/c52dc440c390b6cfa87be9432b141a956e2d56d9b9f5fc8bd71c5f471722/coverage-7.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50913d4bf5ddafa6ca3693da5e4dd833dd1b772e0283c99ca7f7d287db67331a", size = 220539, upload-time = "2026-07-02T13:08:19.252Z" }, - { url = "https://files.pythonhosted.org/packages/3f/26/602de8c2aec7e2e3e99ebfb8e04ba65598f746275396eea5f6794ff4673f/coverage-7.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:359e141ccd33893ce3f1ad5525f8b96083003677c82182e5907d62d4ea5799fc", size = 221058, upload-time = "2026-07-02T13:08:21.013Z" }, - { url = "https://files.pythonhosted.org/packages/fc/13/ebab0743138891c1d646d61e247ec29639afcbb6c4e1905e6a0f0c75291a/coverage-7.15.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3200b6204935f928c64b2ca1f923ab8c1acb7c9de45ec61569711b34d25cccaf", size = 247797, upload-time = "2026-07-02T13:08:22.474Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b7/b6ffb9e042aa48dc4144a8a65529affaec8dca0685309353614a2a7386ad/coverage-7.15.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:be616bf61346883b2cfdc5178669647e03531d81ab761a7e378558b7e8bcb628", size = 249626, upload-time = "2026-07-02T13:08:23.803Z" }, - { url = "https://files.pythonhosted.org/packages/9c/06/243ff05b652333d8e3d060c11223efc2723b19cacf6605e433fa686ab5d4/coverage-7.15.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc7bafc3fe1059463a8fdd97ca79972d6e2bf819d775c7d54991b5b1971201d6", size = 251493, upload-time = "2026-07-02T13:08:25.397Z" }, - { url = "https://files.pythonhosted.org/packages/d3/2b/867faa17030a806114dae388b32a3fa929d8cd4bf39226fbc11f6e6bb705/coverage-7.15.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b713aa7fcf325a01d4184d848acb46fd84f78fdb0978470c636b23a06a753d91", size = 253406, upload-time = "2026-07-02T13:08:26.842Z" }, - { url = "https://files.pythonhosted.org/packages/94/c0/d789ce18f6605afc4895db75723424be2ef494282f77f61d8e5832923183/coverage-7.15.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e38e6fba2d56652fdfaf0231f8f78aeb805234a912de25dc291ee5cce5b8faa4", size = 248512, upload-time = "2026-07-02T13:08:28.398Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b6/b2673c30739f4a2e06649a0a38ad8b093c4d865462dc7bab0e9524a2c3b1/coverage-7.15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:884499f42e382675be80770391983b90e0c0c774d87dbeeebf5f991cf6612b20", size = 249532, upload-time = "2026-07-02T13:08:29.731Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/acd79e9a41beabee92b623afe4f30b549916f48566271475f2907e752828/coverage-7.15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:840481b12e083dbcbafab14794a8781a958edf327c8d3d70b4eee42f9b8253aa", size = 247537, upload-time = "2026-07-02T13:08:31.173Z" }, - { url = "https://files.pythonhosted.org/packages/12/d4/2d301c4d1b3238d7c88b70ab9d13fd53ed9505662a7ff1b46ba1e2e4e3c3/coverage-7.15.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:276646e9481703d09f854f3b2f018f24e19fd7049ae670a92570043eb97203b1", size = 251348, upload-time = "2026-07-02T13:08:32.63Z" }, - { url = "https://files.pythonhosted.org/packages/35/bb/c67708b2bc00f32e12805ec23d5fa677a0a51652f449341a89f9d6b1b715/coverage-7.15.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4de4b4d3f5545aa6c60dc4efd9c63b5b5dcc3bf00fe83146b2bdfffb8f6613bd", size = 247806, upload-time = "2026-07-02T13:08:33.931Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6c/57c4f653c47a6e917748f8938e389e72fbcae44e3643cd906664f0477a13/coverage-7.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5c504097b2a89b1e85bc6070d920df77daec701337e3aeef2c17775a5dd0ca90", size = 248410, upload-time = "2026-07-02T13:08:35.189Z" }, - { url = "https://files.pythonhosted.org/packages/6c/94/bb083041aef828903668f134273f319f2bd49224962875359c52faa5497f/coverage-7.15.0-cp310-cp310-win32.whl", hash = "sha256:f6e80ed91f98316e86b9c137206b04b2bcfbffccbdff49bd2eb09dddb1cf14e0", size = 222588, upload-time = "2026-07-02T13:08:36.486Z" }, - { url = "https://files.pythonhosted.org/packages/ef/94/a09d8ee618956f626741b0734854bac4425a00e10c0565f5abca64e7e751/coverage-7.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:b3b3e22030f3f6f5e01a5ce69936552a5c0f6992b7698777377b99041961031f", size = 223214, upload-time = "2026-07-02T13:08:37.885Z" }, - { url = "https://files.pythonhosted.org/packages/ae/23/82e910835ef4b8391047025e1d53aa48d66029f444eb8b25373c849bf503/coverage-7.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:003fff99412ea848c0aaebcc78ed2b6ce7d8a1227ed17e68470672770b78a02a", size = 220662, upload-time = "2026-07-02T13:08:39.205Z" }, - { url = "https://files.pythonhosted.org/packages/6d/0d/c7b213dde2f1579de5231062b386d8413f79c11667eb58c39319b25991da/coverage-7.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5cbd804bf2784ce7b45114516050f346ecd50f960c4bb630a7ee9e1d78fa2118", size = 221168, upload-time = "2026-07-02T13:08:40.471Z" }, - { url = "https://files.pythonhosted.org/packages/33/77/d000aeedfac085088337b3c7becdad328474b1f8a9e4c9368a0c99605d68/coverage-7.15.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8773e15c23305b58882a4611fb9b2755977eae0dc2e515366a1b6c98866cc4c2", size = 251587, upload-time = "2026-07-02T13:08:42.033Z" }, - { url = "https://files.pythonhosted.org/packages/cc/e0/86787c56b9df17afd370d5e293515dd4d9a107a561d13054873eefad8ecc/coverage-7.15.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f50e40081494c1dc4239ebb202014cbcc3306ea96fb6302a34c8cc0967fc5ae8", size = 253497, upload-time = "2026-07-02T13:08:43.387Z" }, - { url = "https://files.pythonhosted.org/packages/3f/02/181bc917359299c07dead6270f94e411151c8b60cec905c33499da69afe6/coverage-7.15.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:daf96f37f5fc3a7b6c6da862eb4aee61c426bd63da236ed4a73ef0e503b4bca5", size = 255607, upload-time = "2026-07-02T13:08:44.897Z" }, - { url = "https://files.pythonhosted.org/packages/b9/35/ca5e7427699913da6788c4f910e73ab16c5f4b59ec5d3a999dce2a45112f/coverage-7.15.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:51aa20f6ae2788fd197747766edf4cd8234fd9423309b934257fa6b21a592723", size = 257563, upload-time = "2026-07-02T13:08:46.334Z" }, - { url = "https://files.pythonhosted.org/packages/0b/4d/b8220bacc2fc3c4e9078e27c32e99fb411479a4718a72bdd00036a9891c8/coverage-7.15.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:03d1f922757662eb7af586e77834792274cff776bc7b1d1a0b66a49ea9d84735", size = 251726, upload-time = "2026-07-02T13:08:47.941Z" }, - { url = "https://files.pythonhosted.org/packages/c4/e4/2e145da1991d72189b9c3cf7eca05c716ee7080d099aaea6757cfc7df008/coverage-7.15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a6d6acc9a7666245e6133dd15144ca038a85a9cd5026bb06d6bbae9e77440dc9", size = 253301, upload-time = "2026-07-02T13:08:49.5Z" }, - { url = "https://files.pythonhosted.org/packages/72/28/d2c841d698bf762e481f08bd4839d370246b6d9b61dab085a7b20b201a08/coverage-7.15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1ac2c4c27c7df851dc9a017c2d7de00b69147e84ba3d96f37a530b0b6fb51035", size = 251361, upload-time = "2026-07-02T13:08:51.304Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ed/55d9ffde994fba3897c0c783f77a7d053b0c18787f6892ed5b0aed73f469/coverage-7.15.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b761a1d504fd4bd1f20f418753964dca9f5862a511fc854dac58296b3b223671", size = 255129, upload-time = "2026-07-02T13:08:52.661Z" }, - { url = "https://files.pythonhosted.org/packages/1d/c0/ecbf33b8c460ea2718aeb813e2df8140d0370e5f67261c31524ceb0a2a8d/coverage-7.15.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e43b045e11c16e897895758ae90e4a90cf99e93d58549e2f90c0e2272e155695", size = 251081, upload-time = "2026-07-02T13:08:54.188Z" }, - { url = "https://files.pythonhosted.org/packages/a9/de/fb87b4261f54448dd2b9504ef19a58be42cef0d9520595fbfe1219b15234/coverage-7.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:589b54513e901739f4b4582c705ce96b80c96f57641b1464607e2367a270e540", size = 251988, upload-time = "2026-07-02T13:08:55.726Z" }, - { url = "https://files.pythonhosted.org/packages/df/27/3494d5f291b9a4cb868f73c11221a8bd2d5bd761a8f9acea61ff57128dd1/coverage-7.15.0-cp311-cp311-win32.whl", hash = "sha256:106781b8482749162d0b47056937ba0933508e5d9447f65a5e7d5c422f0d6bb4", size = 222754, upload-time = "2026-07-02T13:08:57.091Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ee/cd4847ebc9be6a9c0123d763645a6f1f3be6b8c58c962706368b79cbac07/coverage-7.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:821e92b3631d762a339695824cadbbc73020354eba2a23a551a99ad34938fbe6", size = 223225, upload-time = "2026-07-02T13:08:58.594Z" }, - { url = "https://files.pythonhosted.org/packages/57/37/5011581aa7f2be498b97dcc7c9902192442a42f4f9a748aeadb3d6506b42/coverage-7.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:309990eb5fb8014b9f67cb211f7fd41876ec8a88a88d3ae76de0ed1d611e3640", size = 222774, upload-time = "2026-07-02T13:09:00.074Z" }, - { url = "https://files.pythonhosted.org/packages/2a/74/fd4c0901137c4f8d81a76ada99e43c65163b4c94a02ece107a4ec0c6b615/coverage-7.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b75ee5e8cb7575636ac598719b4307ac529ec8fcd79608a35c3cd4d4dada812d", size = 220838, upload-time = "2026-07-02T13:09:02.084Z" }, - { url = "https://files.pythonhosted.org/packages/0f/2e/2347583467bd7f0402635101a916961915cc68fce652cd0db5f173ea04fc/coverage-7.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffb31267816b93b075302248cc1737506081b4f163df4401e9df1a6424aafabe", size = 221197, upload-time = "2026-07-02T13:09:03.617Z" }, - { url = "https://files.pythonhosted.org/packages/f0/17/99fa688541ae1d6e84543a0e544f83de0c944815b63e9e7b1ed411d15036/coverage-7.15.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e4d0bb73455bf97ab243a8f12c37c686ccf1c13bb614b7b85f1d062f06f42b2c", size = 252705, upload-time = "2026-07-02T13:09:05.059Z" }, - { url = "https://files.pythonhosted.org/packages/fb/02/6a95a5cd83b74839017ef9cf48d2d8c9ae60af919e17a3f336e6f9f1b7bd/coverage-7.15.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:20d9ccc4ebd0edc434d86dfd2a1dd2a8efa6b6b3073d0485a394fee86459ebb4", size = 255441, upload-time = "2026-07-02T13:09:06.559Z" }, - { url = "https://files.pythonhosted.org/packages/67/f2/406f6c57d600f68185942422c4c00f1a3255d60aee6e5fd961425cd9987e/coverage-7.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20c8a976c365c8cb12f0cbd099508772ea41fb5fa80657a8506df0e11bd278c5", size = 256556, upload-time = "2026-07-02T13:09:08.197Z" }, - { url = "https://files.pythonhosted.org/packages/74/8e/d3fa48489c15ecdec1ba48fd61f68798555dddd2f6716f9ad42adeb1a2a9/coverage-7.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f948fd5ba1b9cbca91f0ae08b4c1ce2b139509149a435e2585d056d57d70bf01", size = 258815, upload-time = "2026-07-02T13:09:09.691Z" }, - { url = "https://files.pythonhosted.org/packages/47/2e/2d40ddd110462c6a2769677cf7f1c119a52b45f568978fc6c98e4cc0dd0f/coverage-7.15.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f58185f06edf6ad68ec9fb155d63ef650c82f3fbd7e1770e2867751fb13158f4", size = 253117, upload-time = "2026-07-02T13:09:11.212Z" }, - { url = "https://files.pythonhosted.org/packages/51/c0/310782f0d7c3cb2b5ac05ba8d205fe91f24a36f6bf3256098f1782181c38/coverage-7.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02adc79a920c73c647c5d117f55747df7f2de94571884758ce8bc58e04f0a796", size = 254475, upload-time = "2026-07-02T13:09:13.029Z" }, - { url = "https://files.pythonhosted.org/packages/86/f7/702da6c275f8ae6ade423d2877243122932c9b27f5403003b9ef8c927d12/coverage-7.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6eb7c300fbed667fd6e3588eba71c1904cdb06110ca6fdf908c26bdd88b8e382", size = 252619, upload-time = "2026-07-02T13:09:14.699Z" }, - { url = "https://files.pythonhosted.org/packages/fb/84/c5b15a7e5ecba4e56218d772d99fe80a63e63f8d11f12783723a6005ab45/coverage-7.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b5fb23fa2de9dce1f5c36c09066d8fcda16cd96e8e26686caa2d7cb9b567d65c", size = 256689, upload-time = "2026-07-02T13:09:16.103Z" }, - { url = "https://files.pythonhosted.org/packages/95/2f/c8b07559b57701230c61b23a953858c052890c12ef568d81780c6c46e92e/coverage-7.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cec79341dbe6281484024979976d0c7f22beae08b4a254655decd25d42cbe766", size = 252189, upload-time = "2026-07-02T13:09:17.828Z" }, - { url = "https://files.pythonhosted.org/packages/6b/80/6d2f049dd3fd3dbfd60b62ba6b2162a04009e2c002ce70b24cf3878dec7a/coverage-7.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c664c5444b1d970b1b2a450e21fb19ee5c9cfdf151ded2dda37260031cca0da", size = 254059, upload-time = "2026-07-02T13:09:19.304Z" }, - { url = "https://files.pythonhosted.org/packages/ce/92/b0287a2c42031d25c628f815f89a3cd9f8268ee78bb1252c9356cda1c689/coverage-7.15.0-cp312-cp312-win32.whl", hash = "sha256:5f764a3fa339bde6b3aa97657f5a6a3a9451e4a5b4ea98a2892c773a43525f77", size = 222893, upload-time = "2026-07-02T13:09:20.812Z" }, - { url = "https://files.pythonhosted.org/packages/a9/69/e34c481915fecb499b3146975061dac528752e37706edc1804f32c822469/coverage-7.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:52f9a4d2c4c56c8848bc2f524916698354b0211488b38c49ad9ae54f6cafbff6", size = 223429, upload-time = "2026-07-02T13:09:22.315Z" }, - { url = "https://files.pythonhosted.org/packages/fe/98/6e878f0b571d32684ef3f38d7c03db241ca5b82a5da8a5391596a8f209c4/coverage-7.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:31e5c3e70c85307ea35a12964e2e40f56ca2ee4b1c8c721ccf4609d17071080b", size = 222810, upload-time = "2026-07-02T13:09:23.812Z" }, - { url = "https://files.pythonhosted.org/packages/76/04/145a3748098bcc86b631a85408d2c3dc5c104e0bd86d605468239b25b6c4/coverage-7.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5be4caf3b28836f078abe700f8944dac4a65d78f16d6c600c89cb624e5535782", size = 220863, upload-time = "2026-07-02T13:09:25.371Z" }, - { url = "https://files.pythonhosted.org/packages/a4/5c/4ed55708fed2c64b63c9bc5715daef670872202101938869b7fe5d5fbb8f/coverage-7.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dd58ad1404704303ca8d4f4b8a1095e7cbc7040ef17a66df1e6619aa10176430", size = 221230, upload-time = "2026-07-02T13:09:26.897Z" }, - { url = "https://files.pythonhosted.org/packages/7b/19/3a80b97d3b2a5c77a01ae359c6bed20c13738fe3d9380f08616d4fec0281/coverage-7.15.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bbcbb317c2e5ded5b21104af81c29f391be2af98d065693ffbe8d23949b948e5", size = 252227, upload-time = "2026-07-02T13:09:28.543Z" }, - { url = "https://files.pythonhosted.org/packages/a1/fa/b70062750686bd7da454da27927622f48bbac6990ac7a4c4a4653e7b0036/coverage-7.15.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:27f31ecb458da3f859aab3f15ada871eb7a7768807d88df4a9f186bb17737970", size = 254823, upload-time = "2026-07-02T13:09:30.177Z" }, - { url = "https://files.pythonhosted.org/packages/a9/09/dad6a75a2e561b9dc5086a8c5257a7591d584246f67e23e70d2995b89ab6/coverage-7.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fb759be317fdc62e0f56bffdf61cfcb45c7761ad6b71e3e583e71a67ae753c", size = 256059, upload-time = "2026-07-02T13:09:31.979Z" }, - { url = "https://files.pythonhosted.org/packages/e6/e7/b5d2941fa9564573d44b693a871ff3156f0c42cbefe977a09fa7fdc59971/coverage-7.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5cf007add5ab4bb8fa9f4c77e3732127c9e6cad501d7db43355fbfafca0be84", size = 258190, upload-time = "2026-07-02T13:09:34.035Z" }, - { url = "https://files.pythonhosted.org/packages/7c/1d/8e895bcde3c57ccd46d896dda5f2b3d5df761a1b0c6c9d450d175dedc632/coverage-7.15.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc78d9843bd576fbe2118248258d485e968dc535f95ed504a7b0867ba9b51389", size = 252456, upload-time = "2026-07-02T13:09:35.765Z" }, - { url = "https://files.pythonhosted.org/packages/14/4c/f6997da343ddeb959be82c3b05322793f92c071ad45f7cb8a96336e2dd5f/coverage-7.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a263060f1de0b4b74b4e089c2a70b8003b3781c733329a9c8fd54995328f9950", size = 254192, upload-time = "2026-07-02T13:09:37.445Z" }, - { url = "https://files.pythonhosted.org/packages/17/27/a0bc09d032267b9da89d95a2d874cfbef2a5aebbf0e87cf7aba221d79a99/coverage-7.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c48decf16e0dfd5b049c7d5e82200c23c08126719142998d4f172444e3d0529e", size = 252153, upload-time = "2026-07-02T13:09:39.422Z" }, - { url = "https://files.pythonhosted.org/packages/54/c0/77fc233d9fba07b244c40948c53fe27308b8f21732fb3417f87fbd6fd992/coverage-7.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:08fb028000ed0aaa0a4cbdfbb98be7cb42f370db973fbbb469733505ab20e13e", size = 256310, upload-time = "2026-07-02T13:09:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/d5/24/601cecfb5825becacb8d45219a018a3b55b9dbaec624efdb0ea249d08be2/coverage-7.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb7dc0c3b7d8a1077abea0b8546ebc5e26d6ef6ecefc2f0f5ad2b8a53bdad837", size = 251974, upload-time = "2026-07-02T13:09:42.733Z" }, - { url = "https://files.pythonhosted.org/packages/47/1e/6f45e5a5b3d5484318d368702af6716b5ab8913b0428bec981a562fcf296/coverage-7.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cb3602054ccbe9f0d8c2dc04bbeba90d5719236e2cd06e042ddd6d3fc7b6e37", size = 253745, upload-time = "2026-07-02T13:09:44.376Z" }, - { url = "https://files.pythonhosted.org/packages/8e/db/4df027a77bd11d0e527f44c53557c76e54ad027413d0304252ea3a78d67e/coverage-7.15.0-cp313-cp313-win32.whl", hash = "sha256:0bf781da64326b677be344df505171435b6f58716108606621d5d27d964fff8b", size = 222902, upload-time = "2026-07-02T13:09:46.122Z" }, - { url = "https://files.pythonhosted.org/packages/a0/10/0355894d34e231f2c5449e71287e81a50793a325df2e2b027b7bcd9dfd19/coverage-7.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:2c57a275078ee3fa185f83e400f765bc764a549de66d99b47881645cbd4ea629", size = 223444, upload-time = "2026-07-02T13:09:47.687Z" }, - { url = "https://files.pythonhosted.org/packages/06/ef/bb725f263befaaff851203ab338e68af15e195d7f7b5f323162532d9b6a8/coverage-7.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:3812c61afc6685c7999b39320779ab8f43b7a3081fdb0def39976e56fbdb9a21", size = 222839, upload-time = "2026-07-02T13:09:49.717Z" }, - { url = "https://files.pythonhosted.org/packages/4f/9c/1e3ca54f72a3185ece06c58d871099898c48f0ed6430d17b6ab75f0d180a/coverage-7.15.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:41cb79af843222e11da87127ad0ecbfa878abadd0f770a4a99391a27d3887324", size = 220906, upload-time = "2026-07-02T13:09:51.339Z" }, - { url = "https://files.pythonhosted.org/packages/09/37/f718613d83b274880382f6b67e78f3802549ae39b0b3e65ae5b5974df56e/coverage-7.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7d2008989ef8fe54188d3f3bfa2e3099b025af11e90a6a1b9e7dc433d04263d8", size = 221239, upload-time = "2026-07-02T13:09:53.138Z" }, - { url = "https://files.pythonhosted.org/packages/a7/ce/22bae91e0b75445f68d365c7643ed0aa4880bbf77450ee74ca65bdae53a7/coverage-7.15.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:769e8ece11a596315ebf5aa7ec383aeeed016c091d2bf6363ffb996d41529092", size = 252286, upload-time = "2026-07-02T13:09:54.996Z" }, - { url = "https://files.pythonhosted.org/packages/dd/1e/bec5e32aa508615d9d7a2790effb25fb4dc28606e995816afe400b25ece3/coverage-7.15.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:65a6b6164ee5c39e2f3803f314292d6c61a607ba7fee253d1e03c42dc3903502", size = 254789, upload-time = "2026-07-02T13:09:56.678Z" }, - { url = "https://files.pythonhosted.org/packages/17/29/0e865435b4354e4a7c03b1b7920046d31d0a273d55decefea27e011cb9bf/coverage-7.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75128817f95a5c45bb01d65fd2d8b9cb54bbe03d81608fb70e3e14b437ad56c2", size = 256135, upload-time = "2026-07-02T13:09:58.343Z" }, - { url = "https://files.pythonhosted.org/packages/84/ff/33a870b58a13325d62fc0a6c8f01fa0ff667cef60c7498e2382a147dfa18/coverage-7.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9887bb428fe2d4cd4bee89bac1a6c9932f484afd5b36fbd4ff6ea5f825bb1f5e", size = 258449, upload-time = "2026-07-02T13:10:00.057Z" }, - { url = "https://files.pythonhosted.org/packages/18/7b/6fffe596bf3ddba8462758d02c5dad730fd91055a6634aa2e4226229181a/coverage-7.15.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0bfc0be1f702042207a93a00523b1065ee1fe951e96edf311581c0bbc2e34888", size = 252313, upload-time = "2026-07-02T13:10:01.946Z" }, - { url = "https://files.pythonhosted.org/packages/58/1b/11468dd6c1676ab831a70cb9a8d4e198e8607fa0b7220ab918b73fe9bfbd/coverage-7.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f64627d55def5a43282d70e08396672692f77e4da610a5bb8bb4060b432b6859", size = 254142, upload-time = "2026-07-02T13:10:04.065Z" }, - { url = "https://files.pythonhosted.org/packages/79/41/29328e21d16b1b95092c30dd700e08cf915bd3734f836df8f3bdb0e8fa9f/coverage-7.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:2c6f0fa473003905c6d5bac328ee4eba9fbea654f15bc24b8a3274b23363fa99", size = 252108, upload-time = "2026-07-02T13:10:06.11Z" }, - { url = "https://files.pythonhosted.org/packages/9b/de/05ccfb990439655b35afbfd8e0d13fe66677565a7d4eb38c3f5ef2635e1c/coverage-7.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2bcf9afaf064172c6ec3c58a325a9957ad1178c05dd934e25f253321776e0676", size = 256385, upload-time = "2026-07-02T13:10:08.141Z" }, - { url = "https://files.pythonhosted.org/packages/51/0e/486828a3d2695ea7a2609f17ff572f6b01905e608379440a11da4b8dffbe/coverage-7.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:baf06bc987115d6fb938d403f7eab684a057766c490367999a2b71a6883110c6", size = 251923, upload-time = "2026-07-02T13:10:10.179Z" }, - { url = "https://files.pythonhosted.org/packages/18/c7/03582b6715f078e5e558354c87616d945b9894cda2dace8e4009b17035e4/coverage-7.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0405f2ff97b1c4c0e782cb32e02f32369bcf2e6b618b591d67e1ea754575dfe", size = 253580, upload-time = "2026-07-02T13:10:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/db/dc/9e578bbaf2ecb4959a81b7e7601ad8cca772cba2892e8d144cb749b4a71a/coverage-7.15.0-cp314-cp314-win32.whl", hash = "sha256:ab282853ed5fbd64bbb162f19cb8fcb7087187508a6374b4f9c34ec1577c4e8f", size = 223107, upload-time = "2026-07-02T13:10:13.994Z" }, - { url = "https://files.pythonhosted.org/packages/ae/3e/c8c3b75d8dbe0e35f7b0cc3ff5e949fc59500f70b21d0398813f66740664/coverage-7.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bb3040e9f4bbe26fcb0cd7cc85ac63e630d3f3a9c74f027abf4caa27e706663", size = 223597, upload-time = "2026-07-02T13:10:15.906Z" }, - { url = "https://files.pythonhosted.org/packages/cd/bc/3cbc9fb036eb388519bccd521f783499c39b64256013fbc362782f196fe1/coverage-7.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:346771144d34f7fa84ec28386f78e0f31653f33cf35e19d253d5b35f9e8201da", size = 223020, upload-time = "2026-07-02T13:10:17.844Z" }, - { url = "https://files.pythonhosted.org/packages/28/00/199c4a8d656dff63102577a056c0fce2ff6a79e40adac092fc986c49cbf1/coverage-7.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d34a010905fb6401324ba016b5da03d574967f7b21ce48ea41e66f0f1f95f641", size = 221638, upload-time = "2026-07-02T13:10:19.703Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8e/9d0092c96a3d3a26951ecc7020826aa57bcb1b119ca81acbba996884ab13/coverage-7.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bb25d825d885ca8036795dacfc3924d33091fc76d71ebc99420c6b79e77d96fa", size = 221903, upload-time = "2026-07-02T13:10:21.514Z" }, - { url = "https://files.pythonhosted.org/packages/6d/b4/c0ca3028f42c9a08e51feb4561ef1192e5de99797cd1db5b04590c215bda/coverage-7.15.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:94c9686bfe8a9a6810297aecbd99beaa3445f9e8dc2f80b1382cca0d86b64461", size = 263267, upload-time = "2026-07-02T13:10:23.261Z" }, - { url = "https://files.pythonhosted.org/packages/5f/aa/a375e3846e5d3c013dc600b2a3231089055c73d77f5393dd2192a8d64da6/coverage-7.15.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bd671c25f9d85f09d7ec481d0e43d5139f486c06a37139847a7ce569788af72", size = 265390, upload-time = "2026-07-02T13:10:25.152Z" }, - { url = "https://files.pythonhosted.org/packages/92/e1/5783cdabb797305e1c9e4809fea496d31834c51fa772514f73dc148bcfc9/coverage-7.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:110cbdf8d2e216577312cf06ccf85539c0e5a5420ef747e4a4719b5e483c88cd", size = 267811, upload-time = "2026-07-02T13:10:27.249Z" }, - { url = "https://files.pythonhosted.org/packages/85/31/96d8bbf58b8e9193bc8389574a91a0db48355ee98feb66aa6bf8d1b32eea/coverage-7.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c5d4619214f1d9993e7b00a8600d14614b7e9d84e89507460b126aa5e6559e5", size = 268928, upload-time = "2026-07-02T13:10:29.242Z" }, - { url = "https://files.pythonhosted.org/packages/5e/7a/5294567e811a1cb7eda93140c628fa050d66189da28da320f93d1d815c73/coverage-7.15.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:781a704516e2d8346fbbd5be6c6f3412dd824785146528b3a01816f26c081007", size = 262378, upload-time = "2026-07-02T13:10:31.107Z" }, - { url = "https://files.pythonhosted.org/packages/69/3f/3f48538421f899f28946f90a3d272136a4686e1abf461cc9249a783ee0f3/coverage-7.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd4a1b44bcb65ee29e947ac92bbee04956df3a6bfc6143641bb6cae7ede00fc9", size = 265263, upload-time = "2026-07-02T13:10:32.942Z" }, - { url = "https://files.pythonhosted.org/packages/ce/d3/092df15efcab8a9c1467ee960eb8019bbad3f9300d115d89ea6195f369ff/coverage-7.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0e4950c9d6d3e39c64c991814ff315e2d0b9cb8152363594212c9e55208c0a8f", size = 262866, upload-time = "2026-07-02T13:10:35.104Z" }, - { url = "https://files.pythonhosted.org/packages/e5/ab/0254d2b88665efb2c57ad368cc77ab5de3435bd8d5add4729c1b0e79431e/coverage-7.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:fe9c87ff42e5472d80d21704972e1f96e104a0a599d77c5e35db5a3c562e2571", size = 266599, upload-time = "2026-07-02T13:10:37.05Z" }, - { url = "https://files.pythonhosted.org/packages/a8/79/1cfa4023e489ce6fbc7be4a5d442dbc375edb4f4fda39a352cedb53263c2/coverage-7.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f00d5ae1dd2fe13fb8186e3e7d37bcbd8b25c0d764ff7d1b32cef9be058510a8", size = 261714, upload-time = "2026-07-02T13:10:38.966Z" }, - { url = "https://files.pythonhosted.org/packages/b7/eb/fee5c8665656be63f497418d410484637c438172568688e8ac92e06574e7/coverage-7.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:363ab38cc78b615f11c9cac3cf1d7eef950c18b9fdedfb9066f59461dcf84d68", size = 264025, upload-time = "2026-07-02T13:10:40.789Z" }, - { url = "https://files.pythonhosted.org/packages/ab/99/63005db722f91edc81abc16302f9cc2f6228c1679e46e15be9ae144b14d0/coverage-7.15.0-cp314-cp314t-win32.whl", hash = "sha256:54fd9c53a5fafff509195f1b6a3f9be615d8e8362a3629ff1de23d270c03c86b", size = 223413, upload-time = "2026-07-02T13:10:42.597Z" }, - { url = "https://files.pythonhosted.org/packages/c1/e8/2bc6181c4fb06f1a6b981eb85330cc57bfad7e3f710fc9c9d350013ba228/coverage-7.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:87b47553097ba185ed964866078e7e63adea9f5f51b5f39691c34f30afd21080", size = 224245, upload-time = "2026-07-02T13:10:44.47Z" }, - { url = "https://files.pythonhosted.org/packages/79/b8/4d959bf9cc45d0cfed2f4d35cafcab978cdb6ea02eb5100009cd740632a3/coverage-7.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aeefb2dd178fe7eee79f0ad25d75855cb35ee9ed472db2c5ea06f5b4fd00cec5", size = 223558, upload-time = "2026-07-02T13:10:46.368Z" }, - { url = "https://files.pythonhosted.org/packages/52/30/21b2ad45959cd50e909e02ebac1e30b4ceb7162e91c11d4c570223a458b7/coverage-7.15.0-py3-none-any.whl", hash = "sha256:56da6a4cbe8f7e9e80bd072ca9cefe67d7106a440a7ec06519ec6507ac94ad19", size = 212632, upload-time = "2026-07-02T13:10:48.641Z" }, +version = "7.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/83/6051c2a2feab48ae5bd27c84ef047191d2d4a3172f689e38eaa48ed17db1/coverage-7.15.1.tar.gz", hash = "sha256:165e9949eaf222ef1f018635d0d7f368a23bfe0212af558534c40d8c04686d67", size = 927640, upload-time = "2026-07-12T20:58:19.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/76/eef242d6bb95fb737fefd9bbf3a318897e4e82cf543b53a61c2a6e8588eb/coverage-7.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:05d87c2a43373ad6b976d0a99ad58c48633633bcdeb896dc645a006472cc4a71", size = 221195, upload-time = "2026-07-12T20:55:55.82Z" }, + { url = "https://files.pythonhosted.org/packages/89/73/41cd50da59d513a30fd3a34b5516b962ac17514defdb6705948446a5c0b1/coverage-7.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2afce82f2cf8f4c9002746a42755e1dc61baff33d9f7ab5569b4c9101f8f4d1d", size = 221714, upload-time = "2026-07-12T20:55:58.104Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/6df6ac6677719a087c839208186525b7b1f3653c50196c43246482ada65e/coverage-7.15.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0a545ef5384d787d0fcac6c349afc2c5f99dcc39e13ed3c191b2c06305f64c04", size = 248454, upload-time = "2026-07-12T20:55:59.336Z" }, + { url = "https://files.pythonhosted.org/packages/9f/f5/86f92c429fbf942986d01c3bb7b0b6c3ad06b09f7fa745023877413dde83/coverage-7.15.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:afaa144b8f5b3bc69fe0ce50d401c46b01ab264782553bfd05a3f98804524ecb", size = 250282, upload-time = "2026-07-12T20:56:00.589Z" }, + { url = "https://files.pythonhosted.org/packages/47/c6/99d0f1c1dd1583aaf8c1deca447e44426be54a76027a8c55e29970f22b15/coverage-7.15.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e1b6099490e5f88569c46b18605f556c3b30acc9a0a219cf7ef8fab8f7161ec4", size = 252149, upload-time = "2026-07-12T20:56:01.853Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c3/5d30740ec96f5fd8a659e46b6ee9224197f87aa66ce4a20e93dcf46221ac/coverage-7.15.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bb5f6c2cf1ffd0bf2bd925c7cdcae9b4f208e9696d453ed51eb1f5fa0cc5b45b", size = 254061, upload-time = "2026-07-12T20:56:03.277Z" }, + { url = "https://files.pythonhosted.org/packages/71/e5/3afb2950673ca6cd22bc9ad6a9d6058c853bef8fe27a6d1df3adc274fd88/coverage-7.15.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:81572011fc1fc271317da35da593944daef7bfd507085e35751abbe702b74f69", size = 249152, upload-time = "2026-07-12T20:56:04.618Z" }, + { url = "https://files.pythonhosted.org/packages/ca/53/25ab3c0a7cf517ed4d30cd4dd82331e005d4b57fec8c19c0ffd94fb50baa/coverage-7.15.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8f765d13c08497687d0780cca66115c6aa4ba6703ad43b61e94fab9db689e3a3", size = 250187, upload-time = "2026-07-12T20:56:06Z" }, + { url = "https://files.pythonhosted.org/packages/ad/2a/8afb5f994e26e919e5dca5164f1c7b6b7ce6090aab1aac32b1ae1782f047/coverage-7.15.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:771caba880fee96493d18dfc465c318e08ab74e3bc2a3e4089e52514be6a6e54", size = 248193, upload-time = "2026-07-12T20:56:07.376Z" }, + { url = "https://files.pythonhosted.org/packages/d4/62/b6616e15288c9a7b628f7745e832fd8af2c03ec1f7dc11da25947f5ab16f/coverage-7.15.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:433a73200848e80f27712fc113b6ff5311f29b479a7d3bd4b1106138a77f9674", size = 252004, upload-time = "2026-07-12T20:56:08.787Z" }, + { url = "https://files.pythonhosted.org/packages/9e/21/5d80d65e4df3018dedb23a44f276f20ce26e429ffd76df03de03bcf9a767/coverage-7.15.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:5d6fa45079db9fbeba0a69e3d91189f05301d6ac918162a53179d32fc9ed4910", size = 248463, upload-time = "2026-07-12T20:56:10.245Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/04a22708691561c44d77e1d5a60857b351310fdfbf253533780bb31c8b6f/coverage-7.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:00b6703c6640075cdce5124e9335dfdf9167272475301828acfdd09c0e5ee731", size = 249064, upload-time = "2026-07-12T20:56:11.614Z" }, + { url = "https://files.pythonhosted.org/packages/82/c9/a5c1e8954e657559de478acc1ef73e9393d3fc9b4a2ab4427501d765aca2/coverage-7.15.1-cp310-cp310-win32.whl", hash = "sha256:3ad9a0eac4728327fd870d52f74d2e631d176c5f178eaea2d9983ab5b9755a55", size = 223248, upload-time = "2026-07-12T20:56:13.068Z" }, + { url = "https://files.pythonhosted.org/packages/54/49/5f6566e14611a58e93a926f3a8a7b22e4e0702ebb4c467ac38f452c7f4cf/coverage-7.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:eb5fa75dc3d30e3a1b75da97973479b20ffa9b0641ff56d6e94b5f3e210daa54", size = 223874, upload-time = "2026-07-12T20:56:14.396Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/5095e07a0986930174fc153fd0bb115f7f2724f5344cc672e016d4f23a4e/coverage-7.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6506330b4a8dcf53b95bd84d8d0e817107cdb3fc1438e835029cdf0bc6612eb0", size = 221320, upload-time = "2026-07-12T20:56:16.036Z" }, + { url = "https://files.pythonhosted.org/packages/43/7a/7e2598ce98433a214020a61c0755d5961f9f26df0c630dc73e06b86d3416/coverage-7.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8a51a8ec382c39d939cba0ab07ae949077ae4e842343bd4eed22d432358cea9", size = 221823, upload-time = "2026-07-12T20:56:17.54Z" }, + { url = "https://files.pythonhosted.org/packages/8b/4f/67dac6a69139b490a239d91b6d5ef127982ffb89159769241656ab879ee5/coverage-7.15.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:18ea20e3922d7f8ca9e0ef1084408d08c4ad62d5e531cb9c1f6896a99297ebea", size = 252242, upload-time = "2026-07-12T20:56:18.868Z" }, + { url = "https://files.pythonhosted.org/packages/05/37/5e439725a96de592309725550c8df639c359c209dcb4b152ed46032d4c0d/coverage-7.15.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:aab9902a64b8390e3b56e539fddae1d79a267807fe5cb0c18d7d2f544ce867e2", size = 254153, upload-time = "2026-07-12T20:56:20.118Z" }, + { url = "https://files.pythonhosted.org/packages/0f/1c/671ba9e15f9651a99962fb588a1fe4fb267cae4bb85f9bb4ac4413c6f7ef/coverage-7.15.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cc316264317b07a9e90d7f2b4188a15e36e9b54e651081b791b0515fa612a29", size = 256261, upload-time = "2026-07-12T20:56:21.682Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3c/88305322b4d015b4536b7174b8f9b87f850f01af9d89008cdbf984b65ff2/coverage-7.15.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d4e47e7eea81a8ccf060a07627654151d929da62c7b715738387c200905cec89", size = 258222, upload-time = "2026-07-12T20:56:22.962Z" }, + { url = "https://files.pythonhosted.org/packages/8d/20/d02e42333a57f266ded61ed4fb3821da1f2dc143734aaa3dcc9c117204c5/coverage-7.15.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c6829d9a3b55ad2b73ef5fda8302e5be03683789e88b1a079dcf4a773229c21d", size = 252368, upload-time = "2026-07-12T20:56:24.475Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d0/94ebc010926a191d83e83b1f1236f8bd0d14d0eb98b5c324aa4be2589a8e/coverage-7.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:764e045811f9c8cda436641f3f088283351d331a519b5807f19041cd0a68da1c", size = 253953, upload-time = "2026-07-12T20:56:26.036Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a6/2c8553191da0c2da2c43ff294fb9bab57a5b0fae03560304a82a8b5addc3/coverage-7.15.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:09b3b088aa24489c4082bcc35fcc8224281ab94a653dfb6d3f0c8165b0d628ab", size = 252016, upload-time = "2026-07-12T20:56:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/17/99/20f47986d8360fa1a31d9858dd2ba0be009d44ecfa8d02120b1eb93c028b/coverage-7.15.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7911b02f57053adf8164ae63edb1c26574d24dfccabadc5268cf69310a69a358", size = 255783, upload-time = "2026-07-12T20:56:28.921Z" }, + { url = "https://files.pythonhosted.org/packages/17/36/fb61983b5259f78fa5e62ee74e91fe4de4c556fcbc9a3b9c9021c2f8b57b/coverage-7.15.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:664e279ed40599b8ed16f4db18d92a7e212c73129672bec8f5d96d4da48d2404", size = 251735, upload-time = "2026-07-12T20:56:30.465Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2e/d109d8d2d6c726ba2e78125297073918a2a91464f0ea777e5398fa3cf75c/coverage-7.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34fe7cf79d5f1f87f2e8ce7dc1c32950841f50e10d0120f263856acfad66de34", size = 252645, upload-time = "2026-07-12T20:56:32.076Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a0/5b3219cbb46135e14673427f2bf5890c4da4e6f655243692f3448aa3f42c/coverage-7.15.1-cp311-cp311-win32.whl", hash = "sha256:5e2d2536d2f57a354aa382ed303ac0e2e5c9522a508c05b998d26181b94163a7", size = 223414, upload-time = "2026-07-12T20:56:33.432Z" }, + { url = "https://files.pythonhosted.org/packages/5b/80/24e13ade0b6df8090e2492f9c55044bf1423f7ef28c76fc1af8c827a61cb/coverage-7.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:c337da8fca7ea93ab43f3868cfcde6cf6dad32c3906b273cfbad5d7390bc423b", size = 223888, upload-time = "2026-07-12T20:56:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9e/34659930ae380491af48e2f5f5f9a2e99cd9fb7de3d30f27be5ac5425137/coverage-7.15.1-cp311-cp311-win_arm64.whl", hash = "sha256:db3403fdb7a94d5eb73e099befad8104d2a7d110a0f0d99df0de61c5d1fa756c", size = 223435, upload-time = "2026-07-12T20:56:36.302Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/32c1826309beaf4604c54accef108fdd611e5e5e93f2f5192f050cd5f6bd/coverage-7.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d9476292594309db922cc841dd13b303b3c388f4c25d279884f7e2341c681f80", size = 221497, upload-time = "2026-07-12T20:56:37.628Z" }, + { url = "https://files.pythonhosted.org/packages/db/5c/b88ce0d68fa550c7f3b58617fbf363bce64df5bf8295a01b627e4696e022/coverage-7.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c579056b0de461b3a62318b63d0b6ce90aed7f8158d3f00da094df82f29d189", size = 221854, upload-time = "2026-07-12T20:56:39.033Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/8509fd2a66fc4e0a829f76a0f0b1dc3cc163368352435b5f243168658077/coverage-7.15.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:23214bdbe226f2b0e9c66a7d6a1d59d4a88045dcf86e702cf0fe0d0935e3d615", size = 253359, upload-time = "2026-07-12T20:56:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/e5/81/c7009ed7ea9765adb2b9d095054d748266fae5f07ac6c5f925f33715fcde/coverage-7.15.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:df164be93b46b4825cc39339440a05edc54c4d1d865ba4a60fd43d151a2a1cd3", size = 256096, upload-time = "2026-07-12T20:56:42.115Z" }, + { url = "https://files.pythonhosted.org/packages/21/52/dc8ee03968a5ba86e2da5aa48ddc9e3747bd65d63825fdce2d96acb9c5ff/coverage-7.15.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a524fca1a6f08927d9dc2d4c873cfb7bd7202c247f08b14bdc02424071b8b304", size = 257211, upload-time = "2026-07-12T20:56:43.513Z" }, + { url = "https://files.pythonhosted.org/packages/b8/27/95d7623908da8937deb53d48efcdbf423907a47540e63c62fa21372c652b/coverage-7.15.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d70f3542cd38de85a9e257dcb1ac4c1ab4b6d7d2c2a645809207556628755d1c", size = 259473, upload-time = "2026-07-12T20:56:44.974Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3b/730d761928de97d585465680b568ae69622fb40716babadeabffe75cb51b/coverage-7.15.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d78aa537237212c4313aabe5e964b66acc86350ed19ebc56a3e202df33b6077b", size = 253759, upload-time = "2026-07-12T20:56:46.615Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fc/6b9277acff1f9484b6c12857af5774689d1a6a95e13265f7405329d2f5da/coverage-7.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a318112bb4f79d9d04766196d5a3388caa825908a6a9b052aa87de3d9aea7c61", size = 255131, upload-time = "2026-07-12T20:56:48.073Z" }, + { url = "https://files.pythonhosted.org/packages/3d/f2/c704f86129594ba34e25a64695d2068c71d51c2b98907184d716c94f4aec/coverage-7.15.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e55d24cada901963eed5bc89fa562aa033f0d84b9d3de4ecf363737c13aed11e", size = 253275, upload-time = "2026-07-12T20:56:49.538Z" }, + { url = "https://files.pythonhosted.org/packages/f6/29/80fee8af47de4a6dce71ccf2938491f444687a756af258a56d8469b8f1b0/coverage-7.15.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3c78f0cea7275342cf2adc2ad5fdd0aafa106ad91e66d573568f2fcf62c41df5", size = 257345, upload-time = "2026-07-12T20:56:51.038Z" }, + { url = "https://files.pythonhosted.org/packages/20/21/a1e7d7ed1b48a8adf8fd5154d9e83fcc5ad8e6ff20ae00e44865057dce8d/coverage-7.15.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:86bd37eabe39977216f630a7fc1b698e7f5e81a191c7186013245c6c3d313f9d", size = 252844, upload-time = "2026-07-12T20:56:52.535Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8c/a4bc26e6ee207d412f3678f04d74be1550e83140563ca0e4997510579712/coverage-7.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c6db15c217693bdc3ca0b84de1ba9afafe1c14c26a8a29d77f4ed0de2b6132e2", size = 254716, upload-time = "2026-07-12T20:56:53.968Z" }, + { url = "https://files.pythonhosted.org/packages/11/9d/8ad0266ecfada6353cf6627a1a02294cf55a907521b6ee0bd7b770cfd659/coverage-7.15.1-cp312-cp312-win32.whl", hash = "sha256:359f3fbe09a51500c51966596ee4ee4070b356552c70b3b2420eb200d68e0f76", size = 223554, upload-time = "2026-07-12T20:56:55.583Z" }, + { url = "https://files.pythonhosted.org/packages/81/6d/24224929e06c6e05a93f738bc5f9e8e6ab658f8f1d9b823e7b85430e28b8/coverage-7.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:fa75dc099c126e941a9c0baa8ebd2cbc78bd778687534fe410baf754f6d9e374", size = 224087, upload-time = "2026-07-12T20:56:57.041Z" }, + { url = "https://files.pythonhosted.org/packages/35/23/f81441dd01de88e53c97842e706907b307d9078918c3f4998b11e9ac7250/coverage-7.15.1-cp312-cp312-win_arm64.whl", hash = "sha256:26f89cf6d0634375f454fa71057945ad18edb0f1607a90fecf22c57dc3dc289a", size = 223472, upload-time = "2026-07-12T20:56:58.594Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1e/6fa289d7993a2a39f1b283ddb58c4bfec80f7800be654b8ba8a9f6a07c63/coverage-7.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:71ac4ca1658ca99160fd58cc6967110e989c34b04627f24ed6ec9f70fb24571a", size = 221519, upload-time = "2026-07-12T20:57:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e1/0db4902a0588234a70ab0218073c0b20fbc5c740aa35f91d360160a2ebc9/coverage-7.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:26a40cbf2b13bd94af53ee02a424cb3bb96a9edfac0d00834bd068512a62714b", size = 221895, upload-time = "2026-07-12T20:57:01.867Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cb/3719783865092dac5e08df842730305ee9ab1973ae7ddb6fbdf27d401f30/coverage-7.15.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4c5a5eff4ad4f9f7088fd3fc7a66d98d06566ee294b3b053309fb0a3b45be1e", size = 252882, upload-time = "2026-07-12T20:57:03.459Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5e/caf3abbdbb22629626160ffc9c017eb995b7cb11c0be46b974834cef1792/coverage-7.15.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:962aa56c1c9b016d681265880eb6acc9966029d2c4c559319cc43a1abbb9b59a", size = 255479, upload-time = "2026-07-12T20:57:04.984Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f1/d60f375bfe095fef944f0f19427aefdbf9bdd5a9571c41a4bf6e2f5fdb81/coverage-7.15.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1678eb2dc57a8ce67601b029582ef6d41e9e6ca22692aaeccd4107e40f27386c", size = 256715, upload-time = "2026-07-12T20:57:06.446Z" }, + { url = "https://files.pythonhosted.org/packages/d7/17/8b0cbc90d02dc5adad4d9034c1824ec3fa567771b4c39d9c1e3f9b1431b8/coverage-7.15.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1174900a43f6f8c425fee10d7dbddc308adefcdc78aaced32357f5ab750a0e90", size = 258845, upload-time = "2026-07-12T20:57:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/92/29/c5e69f5fb75c322e9a3e4ef64d02eebfc3d66efceccc8514ff80a3c13a56/coverage-7.15.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:98847557a6859cadf693792ce89f440cb89692993f60dc6d3a7e35f3d340216f", size = 253098, upload-time = "2026-07-12T20:57:09.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/57/21144252fdd0c01d707d48fbcea13a80b0b7c42ced3f299f885ab8978c3a/coverage-7.15.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8697b2edb57143546a24389efc11e1b000cd5800fc20d84f04edb601e4a7cfb8", size = 254844, upload-time = "2026-07-12T20:57:11.141Z" }, + { url = "https://files.pythonhosted.org/packages/59/2a/499a28a322b0ce6768328e6c5bb2e2ad00ac068a7c7adb2ecd8533c8c5d9/coverage-7.15.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6827ac0519be3fe91bf96b4060eb00d1d24f82649b29862cd75a3cfca248b02a", size = 252807, upload-time = "2026-07-12T20:57:12.678Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/928a95da5da8b60f2b00e1482c7787b3316188e6d2d227fb8e124ada43a1/coverage-7.15.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2de8ecbbc77c7e4d22572779920ed8979c69168675e96be3a548c996568c6c31", size = 256965, upload-time = "2026-07-12T20:57:14.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/10/889adbc1b8c9f866ed51e18a98bcafc0259fb9d29b81f50a719407c64ea8/coverage-7.15.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b25f0f0fa5260df9d7bb55d47c8bdc23fa3382c1a18f7c9cae122e6c320b1ad", size = 252628, upload-time = "2026-07-12T20:57:15.892Z" }, + { url = "https://files.pythonhosted.org/packages/1a/30/a5e1871e5d93416511f8e359d1ccebfe0cbb050a1bbf7dd20228533ec0cf/coverage-7.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a2effcbd93ae340a58db718fe4181d967f84d352c4cefeaab4ff82ce813901a", size = 254399, upload-time = "2026-07-12T20:57:17.703Z" }, + { url = "https://files.pythonhosted.org/packages/2d/26/c36fbffd549dadbdd1a75827528fb00a4c46aa3187b007b750b1e2cebbf2/coverage-7.15.1-cp313-cp313-win32.whl", hash = "sha256:895e65c96aef0cecea250f6e35e9a32f11375514e1a0cb5210e0fda128c04e8e", size = 223564, upload-time = "2026-07-12T20:57:19.253Z" }, + { url = "https://files.pythonhosted.org/packages/16/fc/becbb9d2c4206d242b9b1e1e8e24a42f7926c0200dd3c788b9fab4bb96d5/coverage-7.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6d0a28b63a0d75f9ed5118105d1154fc3aa40a8605a30d5d87e3d043ad90fe7", size = 224106, upload-time = "2026-07-12T20:57:21.108Z" }, + { url = "https://files.pythonhosted.org/packages/d3/30/1cfc641461369b6858799fca61c0a8b5edc490c519bf7c636ffa6bbf556f/coverage-7.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:b4ee9818e8bae3544379ad2c09b851c4fb886aaa8860d57a1c1316ddcb16db49", size = 223497, upload-time = "2026-07-12T20:57:22.734Z" }, + { url = "https://files.pythonhosted.org/packages/f0/46/81961952e7aebfb38ad0ae4264e8954cc607a7af9e7ac111f9fa986595cc/coverage-7.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a886af95f59edf67d5770fd3564d53f4a8af93f25f8c1d60d27e00d7f5674ee8", size = 221560, upload-time = "2026-07-12T20:57:24.282Z" }, + { url = "https://files.pythonhosted.org/packages/13/d2/ee14d715889f216baf47301d9f469e08fff6995552aaf67e897b282865f6/coverage-7.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:985657ebd707941de90d488d1cbb5efac20bdf81f7b91eba771624ccda4d36f4", size = 221894, upload-time = "2026-07-12T20:57:25.87Z" }, + { url = "https://files.pythonhosted.org/packages/f3/38/f830bc6e6c2c5f23f43847125e6c650d378872f7eeba8d49f1d42193e8a9/coverage-7.15.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5bbe2a06e0a5e1404d9ffbdb49b819bbd6a3bb198ebea4c8dfe7ad9f1e1c2e81", size = 252938, upload-time = "2026-07-12T20:57:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/e1/53/0d3dd963631259d794c898735d5436e68d6a8d40749c419a07ff7c171469/coverage-7.15.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bde0fe24083d0b7b3dbafa7a09f0796410af1afa2523f28f5f208d8340a4aaca", size = 255445, upload-time = "2026-07-12T20:57:29.234Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fd/aabed228557565c958259251b89bab8c5669b31291fa63b3e2154ebb017a/coverage-7.15.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f89f7453d6d46db14cf233e2cd8edcd78de2b9c49d4f1dc109590b4e5dbfbb74", size = 256790, upload-time = "2026-07-12T20:57:30.826Z" }, + { url = "https://files.pythonhosted.org/packages/bc/aa/1cc888e5d3623e603c4e5399653cb25728bb2b40d7519188a3e293d24620/coverage-7.15.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc3656c9ecc27b36bd0907455b77f83c0069ca9ad4a66dec892b76c696eb6047", size = 259104, upload-time = "2026-07-12T20:57:32.63Z" }, + { url = "https://files.pythonhosted.org/packages/5f/61/fc16d5f5e53098dae41efa21e8ccc611a9b4fe922750dd03dc56db552182/coverage-7.15.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:24d8e85a2a45e44883b488c2659f51fa761dad5353fdb319b672a93facbd2ca9", size = 252956, upload-time = "2026-07-12T20:57:34.316Z" }, + { url = "https://files.pythonhosted.org/packages/a1/f3/52384668c3de4519ca770bf1975a89e4d6eb5aa2faf0da0577a14008cba4/coverage-7.15.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:68931b5fe746ed4fdaa8892989cab9e6c35781eeb3b0ab2ded893d561e1b3652", size = 254797, upload-time = "2026-07-12T20:57:35.947Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/54b807e7c1868178e902fd8360b5d4e559394462f97285c50edf1c4608db/coverage-7.15.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1ce6947e2a95534ecaa5a15e73c21e550514c980d80eda204d064d789a95f6a4", size = 252762, upload-time = "2026-07-12T20:57:37.856Z" }, + { url = "https://files.pythonhosted.org/packages/7b/48/dde8adf0338e3ace738757dccf1ce817e5fdcadfae77e1b48a77e5a3b265/coverage-7.15.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:841befdbc89b9c82435fc25b0f4f41858b6238693e45af758bec4cfc1968171c", size = 257037, upload-time = "2026-07-12T20:57:39.488Z" }, + { url = "https://files.pythonhosted.org/packages/07/f2/179dd88cf60a0aeeee16a970ffe250dccea8b80ed4beab4c5d3f6c41ad4b/coverage-7.15.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5d3de58b837375e7f4c0e1a088ccab5f655efb2fd7427b729df02c862a559633", size = 252577, upload-time = "2026-07-12T20:57:41.363Z" }, + { url = "https://files.pythonhosted.org/packages/2c/37/8a593d69ab521beb6a105a2017cac4ba94425ee0a8349e29c3c0b522d24f/coverage-7.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b1801963f9f44ae0c0f6d737bc7aeb2bbcde7d1fe7e3b43cddc1961af42d3b41", size = 254235, upload-time = "2026-07-12T20:57:43.025Z" }, + { url = "https://files.pythonhosted.org/packages/6d/34/bc9b3bced66f2cdad4bf5e57ae51c54ea226e8aaaebfc9370a9a11877bf3/coverage-7.15.1-cp314-cp314-win32.whl", hash = "sha256:8c7953c4128ef53b6ffb5f90d87c87d4ce26731df294760bb2314eb0e069e44b", size = 223771, upload-time = "2026-07-12T20:57:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f3/4d20337bed61915d14349e62b88d5e4144d5a9872b64adbe90e9906db6db/coverage-7.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:6f0bab60a582d415f0fb535ccff13ba334a47a1538f98913330a525d23bd535a", size = 224257, upload-time = "2026-07-12T20:57:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/7b/df/bbfeae4948f3ded516f92b32f2d57952427fc5ecfc0924487bb6ee6a5f38/coverage-7.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:0f410ee8f0ac4ec7db71bc0b7632a8b9994e1cad2755bd1566c17e6a162caa74", size = 223683, upload-time = "2026-07-12T20:57:48.106Z" }, + { url = "https://files.pythonhosted.org/packages/35/65/0b431856064e387d1f5cf474625e4a0465e907024d42f35de6af19ced0be/coverage-7.15.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:fc868bab88e049d41fcd41766810d790a8b960053be2a45e060f5ce0d31d258b", size = 222298, upload-time = "2026-07-12T20:57:49.882Z" }, + { url = "https://files.pythonhosted.org/packages/a6/96/50eac9bd49df8a3df5f3d38746d1bf332299dffb554486c94ebd55c9dc49/coverage-7.15.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:206d4ec6028f2773b40932d09f074539d6bcdd8f6b318d40cb04bdbd68ed0b49", size = 222561, upload-time = "2026-07-12T20:57:51.688Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5b/6ba1c4a27e10b8816fd2622b98162c83d3bdf1185097360373611bf96364/coverage-7.15.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:620482ef1c9f4e61f962e159325fe77dea59d16e39d9c9470d069053b244d864", size = 263923, upload-time = "2026-07-12T20:57:53.392Z" }, + { url = "https://files.pythonhosted.org/packages/e4/59/fe03ade97a3ca2d890e98c572cf48a99fda9adba85757c34b823f41efe1e/coverage-7.15.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d385fc9b054e309ad3cecdc77b586d2af0c98aeec2fdb3773544586f366e817c", size = 266043, upload-time = "2026-07-12T20:57:55.095Z" }, + { url = "https://files.pythonhosted.org/packages/16/e0/55c4b1217a572a43e13b39e1eb78d0da29fb23679003bd0cdf22c50b1978/coverage-7.15.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1198bca9c0dd7c188aae1f185b0c0b5fc4f0a2b6909000858c29550320bdb07", size = 268465, upload-time = "2026-07-12T20:57:57.017Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/ee47944f76afc03909119b036fe9e0da8cbd274a5141287de79791a0fb6d/coverage-7.15.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5d0297e6a070eadb49df7cddd0ab6f420b8b689dd8904c7dd815a323168fa57e", size = 269584, upload-time = "2026-07-12T20:57:58.958Z" }, + { url = "https://files.pythonhosted.org/packages/cb/8a/6b4d9779c7b2e21c3d12c3425e3261aa7411399319e27aa402dfec4db5d0/coverage-7.15.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:916fcf2214f56960e409561b37fc32a160a42b6e85483d0652d7b70fa55d707e", size = 263019, upload-time = "2026-07-12T20:58:00.979Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1e/db5c7fa0c8ba5ece390a1e1a3f30db71d440240a80589df28e66a7503c40/coverage-7.15.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f837bae572c7869ffaa502e604c87e182543012831cf87aae4586ad090ac6dcf", size = 265916, upload-time = "2026-07-12T20:58:03.005Z" }, + { url = "https://files.pythonhosted.org/packages/83/53/fe5176682b00709b13fab36addd26883139d0dea430816fea412e69255e2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3ea65e3ee6c7c32349fd00559927a9e577bdd72386087eeed1c42b62dfce9b82", size = 263520, upload-time = "2026-07-12T20:58:04.994Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e7/16f15127be93fbc70c667df5ec5dce934fc76c9b0888d84969a5d5341e2c/coverage-7.15.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:345034976f46a1c54bd17f4e43eb30bb92cb7082fcddff03250cff136cc4eb82", size = 267254, upload-time = "2026-07-12T20:58:06.824Z" }, + { url = "https://files.pythonhosted.org/packages/cd/73/e5119111f6f065376395a525f7ce6e9174d83f3db6d217ea0211a61cca4d/coverage-7.15.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4f051a64eb8f8addb4661c2b41d6eea5b7ebc68ad4b2baea8d9bc54e1956e5f7", size = 262366, upload-time = "2026-07-12T20:58:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d7/9c/6d0a81182df18a73b081e7a8630f0e2a52b12dfd7898c6ab839551a454d2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a7625770f7720b49bb30d194ad2f8d50fab3c5177874af3d2399676f95f9c594", size = 264680, upload-time = "2026-07-12T20:58:10.359Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a2/bac0cbd4450638f1be2041a464b1766c8cc94abf705a2df6f1c8d4be870d/coverage-7.15.1-cp314-cp314t-win32.whl", hash = "sha256:81e503d130a472ad1bd38199ecd35116b40d92bcd31e27a2cacde035381f2070", size = 224077, upload-time = "2026-07-12T20:58:12.065Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b2/d83c5403155172a43ba47c08641bad3f89822d8405102423a41339d2c857/coverage-7.15.1-cp314-cp314t-win_amd64.whl", hash = "sha256:724e878b213b302ad46e9f2fc872d386613f20ebfc492a211482d917ea76c14f", size = 224908, upload-time = "2026-07-12T20:58:13.956Z" }, + { url = "https://files.pythonhosted.org/packages/cc/41/442b74cad832cc77712080585455482e7cc4f4a9a13192f65731dcd18231/coverage-7.15.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ce2f05c14d077f406fefc4fa5e4f093ad0e0787549f6582535d6e28766f0361b", size = 224219, upload-time = "2026-07-12T20:58:16.315Z" }, + { url = "https://files.pythonhosted.org/packages/34/98/07a67cf1a26e795d617ed5c540c042b0ac87b72f810c30c07f076cf334f3/coverage-7.15.1-py3-none-any.whl", hash = "sha256:717d01e6e00bed56ad13306f19e0dd2f4f645ee8159d2c72c72301d6cfc7090c", size = 213284, upload-time = "2026-07-12T20:58:18.079Z" }, ] [package.optional-dependencies] @@ -11939,7 +11996,7 @@ wheels = [ [[package]] name = "fastcore" -version = "2.0.4" +version = "2.0.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -11953,9 +12010,9 @@ resolution-markers = [ "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/3a/6ab189da89201ea24cd294bc56a20b6de621b7899485b6497c9240963a5b/fastcore-2.0.4.tar.gz", hash = "sha256:5b65f7fc39ab716c62e9117746c6ea897ae111f9d71f4d7d6d0c08daac8c9480", size = 103215, upload-time = "2026-07-11T04:46:52.234Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/83/a4452a9e0c078d844e0882745d0a44d3f57c968e1ae0d5dbc06892a403d0/fastcore-2.0.5.tar.gz", hash = "sha256:7be70ec5517723e4caeac0725a75942f9f081d152470d1401699a1ba3f536c01", size = 103271, upload-time = "2026-07-13T07:19:35.999Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/40/25/4094eede4ca19d712c646402a9db5c9ab67cba4e6ff192754805655ec8a9/fastcore-2.0.4-py3-none-any.whl", hash = "sha256:15a4044114dc54d1c6bfdfec1a6c0e5b64c9b115d5881e9ef879de2a0f9b9a97", size = 108099, upload-time = "2026-07-11T04:46:50.641Z" }, + { url = "https://files.pythonhosted.org/packages/5e/cc/4bc2c4514444fe27514700a4fb2d8385f32cb65f3f0e0cf752eceed5854a/fastcore-2.0.5-py3-none-any.whl", hash = "sha256:88a7149aea4457717a02bdc74a4e3e4669d1cc9b611759da984333f83a280d85", size = 108123, upload-time = "2026-07-13T07:19:34.53Z" }, ] [[package]] @@ -12493,7 +12550,7 @@ name = "geomet" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "python_full_version < '3.14'" }, + { name = "click" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2a/8c/dde022aa6747b114f6b14a7392871275dea8867e2bd26cddb80cc6d66620/geomet-1.1.0.tar.gz", hash = "sha256:51e92231a0ef6aaa63ac20c443377ba78a303fd2ecd179dc3567de79f3c11605", size = 28732, upload-time = "2023-11-14T15:43:36.764Z" } wheels = [ @@ -12574,14 +12631,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.50" +version = "3.1.51" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +sdist = { url = "https://files.pythonhosted.org/packages/59/30/a8a0c15f9480dc91b5b7f11ebd26105e5f80898d7ff02da197fef35d8395/gitpython-3.1.51.tar.gz", hash = "sha256:22c9c94bb6b0b9f3c7157c684fece45a414cea204586b600beae6cd4570dcd6d", size = 223519, upload-time = "2026-07-12T13:40:07.922Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, + { url = "https://files.pythonhosted.org/packages/f8/11/1232bb1ba52a230f20ff08e1b3df7cd9d43a71b61cc0a1c37941064fe4c7/gitpython-3.1.51-py3-none-any.whl", hash = "sha256:d5b29685708f3c80a56db040b665bfd69492c8e150b5848532af8013b686c5a4", size = 215246, upload-time = "2026-07-12T13:40:06.43Z" }, ] [[package]] @@ -13926,7 +13983,7 @@ name = "gssapi" version = "1.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "decorator", marker = "python_full_version < '3.15' or sys_platform != 'win32'" }, + { name = "decorator" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/52/c1e90623c259a42ab0587078bb04f959867b970add46ff66750ead8fc7c5/gssapi-1.11.1.tar.gz", hash = "sha256:2049ee4b1d0c363163a1344b7282a363f9f4094e51d2c36de0cf01d4735e0ae2", size = 95233, upload-time = "2026-01-26T21:01:39.463Z" } wheels = [ @@ -14711,17 +14768,17 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -14745,18 +14802,18 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -14768,7 +14825,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -18096,9 +18153,9 @@ wheels = [ [package.optional-dependencies] sql-other = [ - { name = "adbc-driver-postgresql", marker = "python_full_version < '3.13'" }, - { name = "adbc-driver-sqlite", marker = "python_full_version < '3.13'" }, - { name = "sqlalchemy", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-postgresql" }, + { name = "adbc-driver-sqlite" }, + { name = "sqlalchemy" }, ] [[package]] @@ -20506,9 +20563,9 @@ name = "python3-saml" version = "1.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "isodate", marker = "python_full_version < '3.13'" }, - { name = "lxml", marker = "python_full_version < '3.13'" }, - { name = "xmlsec", marker = "python_full_version < '3.13'" }, + { name = "isodate" }, + { name = "lxml" }, + { name = "xmlsec" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/98/6e0268c3a9893af3d4c5cf670183e0314cd6b5cb034a612d6a7cc5060df8/python3-saml-1.16.0.tar.gz", hash = "sha256:97c9669aecabc283c6e5fb4eb264f446b6e006f5267d01c9734f9d8bffdac133", size = 83468, upload-time = "2023-10-09T10:37:43.128Z" } wheels = [ @@ -20572,7 +20629,7 @@ dependencies = [ { name = "cryptography" }, { name = "docker" }, { name = "fastcore", version = "1.14.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "fastcore", version = "2.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "fastcore", version = "2.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "httpr" }, { name = "httpx", extra = ["http2"] }, { name = "jinja2" }, @@ -21641,10 +21698,10 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, + { name = "joblib" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/37/59/44985a2bdc95c74e34fef3d10cb5d93ce13b0e2a7baefffe1b53853b502d/scikit_learn-1.5.2.tar.gz", hash = "sha256:b4237ed7b3fdd0a4882792e68ef2545d5baa50aca3bb45aa7df468138ad8f94d", size = 7001680, upload-time = "2024-09-11T15:50:10.957Z" } wheels = [ @@ -21687,13 +21744,13 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11'" }, - { name = "narwhals", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -21738,7 +21795,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -21798,7 +21855,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -21879,7 +21936,7 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -21964,8 +22021,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, - { name = "jeepney", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -22268,15 +22325,15 @@ name = "snowflake-snowpark-python" version = "1.53.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cloudpickle", version = "3.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, - { name = "protobuf", marker = "python_full_version < '3.14'" }, - { name = "python-dateutil", marker = "python_full_version < '3.14'" }, - { name = "pyyaml", marker = "python_full_version < '3.14'" }, - { name = "setuptools", marker = "python_full_version < '3.14'" }, - { name = "snowflake-connector-python", marker = "python_full_version < '3.14'" }, - { name = "typing-extensions", marker = "python_full_version < '3.14'" }, - { name = "tzlocal", marker = "python_full_version < '3.14'" }, - { name = "wheel", marker = "python_full_version < '3.14'" }, + { name = "cloudpickle", version = "3.1.1", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "setuptools" }, + { name = "snowflake-connector-python" }, + { name = "typing-extensions" }, + { name = "tzlocal" }, + { name = "wheel" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/98/38189e919c54fc6f09a43e778d850ebf2116ced491595971ae5f9ca01b0c/snowflake_snowpark_python-1.53.0.tar.gz", hash = "sha256:6eab04d5703fac72982f3666140c006d6fb9964d46a04fd953766410af8fc62e", size = 1783158, upload-time = "2026-07-09T16:15:19.599Z" } wheels = [ @@ -22323,23 +22380,23 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "alabaster", marker = "python_full_version < '3.11'" }, - { name = "babel", marker = "python_full_version < '3.11'" }, - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "imagesize", marker = "python_full_version < '3.11'" }, - { name = "jinja2", marker = "python_full_version < '3.11'" }, - { name = "packaging", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "requests", marker = "python_full_version < '3.11'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.11'" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" } }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } wheels = [ @@ -22355,23 +22412,23 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "alabaster", marker = "python_full_version == '3.11.*'" }, - { name = "babel", marker = "python_full_version == '3.11.*'" }, - { name = "colorama", marker = "python_full_version == '3.11.*' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "imagesize", marker = "python_full_version == '3.11.*'" }, - { name = "jinja2", marker = "python_full_version == '3.11.*'" }, - { name = "packaging", marker = "python_full_version == '3.11.*'" }, - { name = "pygments", marker = "python_full_version == '3.11.*'" }, - { name = "requests", marker = "python_full_version == '3.11.*'" }, - { name = "roman-numerals", marker = "python_full_version == '3.11.*'" }, - { name = "snowballstemmer", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version == '3.11.*'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } wheels = [ @@ -22393,23 +22450,23 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "alabaster", marker = "python_full_version >= '3.12'" }, - { name = "babel", marker = "python_full_version >= '3.12'" }, - { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "imagesize", marker = "python_full_version >= '3.12'" }, - { name = "jinja2", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pygments", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ @@ -22475,12 +22532,12 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "starlette", marker = "python_full_version < '3.11'" }, - { name = "uvicorn", marker = "python_full_version < '3.11'" }, - { name = "watchfiles", marker = "python_full_version < '3.11'" }, - { name = "websockets", marker = "python_full_version < '3.11'" }, + { name = "colorama" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" } }, + { name = "starlette" }, + { name = "uvicorn" }, + { name = "watchfiles" }, + { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/2c/155e1de2c1ba96a72e5dba152c509a8b41e047ee5c2def9e9f0d812f8be7/sphinx_autobuild-2024.10.3.tar.gz", hash = "sha256:248150f8f333e825107b6d4b86113ab28fa51750e5f9ae63b59dc339be951fb1", size = 14023, upload-time = "2024-10-02T23:15:30.172Z" } wheels = [ @@ -22504,13 +22561,13 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "colorama" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "starlette", marker = "python_full_version >= '3.11'" }, - { name = "uvicorn", marker = "python_full_version >= '3.11'" }, - { name = "watchfiles", marker = "python_full_version >= '3.11'" }, - { name = "websockets", marker = "python_full_version >= '3.11'" }, + { name = "starlette" }, + { name = "uvicorn" }, + { name = "watchfiles" }, + { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } wheels = [ @@ -22540,7 +22597,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/2b/69/b34e0cb5336f09c6866d53b4a19d76c227cdec1bbc7ac4de63ca7d58c9c7/sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632", size = 2193689, upload-time = "2024-08-02T13:48:44.277Z" } wheels = [ @@ -22564,7 +22621,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/13/7b/804f311da4663a4aecc6cf7abd83443f3d4ded970826d0c958edc77d4527/sphinx_design-0.7.0.tar.gz", hash = "sha256:d2a3f5b19c24b916adb52f97c5f00efab4009ca337812001109084a740ec9b7a", size = 2203582, upload-time = "2026-01-19T13:12:53.297Z" } @@ -22977,14 +23034,15 @@ wheels = [ [[package]] name = "svcs" -version = "25.1.0" +version = "26.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/08/e1/2d56ec21820cae24a6215dc1d6a224167b0e24368bf53166551064742e0d/svcs-25.1.0.tar.gz", hash = "sha256:64dd74d0c4e8fee79a9ac7550a9d824670b228df390ba1911614e29b59b3f2c2", size = 714086, upload-time = "2025-01-25T13:15:21.505Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/ea/10957367bf49b401a8100ff434f453a340cc30c9c2fc3b2367e6c282f03d/svcs-26.1.0.tar.gz", hash = "sha256:71550e3b228d529448136d78013b2d65528eec4632b8779f8f89c1d8585eed99", size = 906116, upload-time = "2026-07-13T10:09:38.322Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/1c/a6b5d90a9ca479805798276728ccbbdff0c7228e2ea93b1f731779d635e3/svcs-25.1.0-py3-none-any.whl", hash = "sha256:df49cb7d1a05dfd2dd60af1a2cb84b9c3bb0a74728833cb8c54a7ceeecce6c97", size = 19456, upload-time = "2025-01-25T13:15:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8c/5f9441100851bea8f4735cf54a2980d2ad9300a5933c1d5e416448fd789a/svcs-26.1.0-py3-none-any.whl", hash = "sha256:87428c6371af6ae5d0936193613ba4e463118f36b43e15a708f417a8e0b85520", size = 23609, upload-time = "2026-07-13T10:09:36.945Z" }, ] [[package]] @@ -23575,11 +23633,11 @@ wheels = [ [[package]] name = "types-cachetools" -version = "7.0.0.20260518" +version = "7.0.0.20260713" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c8/14/1e4fca2b250dbc75be9f0beab083acb3cd1151711e1031eb4a854dfd71be/types_cachetools-7.0.0.20260518.tar.gz", hash = "sha256:7730014e4fef0c6f01e2cd0f980f8ce6d1b1d2472c8459c1f382348ec1a6b435", size = 10072, upload-time = "2026-05-18T06:02:20.396Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/64/66d7efdb36ecf6826aca5415e59fe2df96e97d24157147e53acfbe8dda11/types_cachetools-7.0.0.20260713.tar.gz", hash = "sha256:f1acf079e9c66a81e096a897ef0b261a82117cf856834e37b4bd0c9a116a076a", size = 10199, upload-time = "2026-07-13T05:22:21.845Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/45/e0/767be6b60859fd2edc4512fabdedbce703fc8d4ec5007b31abaf37a51c6c/types_cachetools-7.0.0.20260518-py3-none-any.whl", hash = "sha256:997b356870915f8bbc9b2cdb4e7271c01d487996fdac2a9c8e91cc5b1261b3d1", size = 9500, upload-time = "2026-05-18T06:02:19.042Z" }, + { url = "https://files.pythonhosted.org/packages/0e/c7/d3525c9dbdc1be7786bad46655ef051b6e7993f656d304719ec40079c91c/types_cachetools-7.0.0.20260713-py3-none-any.whl", hash = "sha256:6db9bcc7a3840d39e91c04117d85a9d0937eacc9d14d12a873e2b01a2d24a71d", size = 9615, upload-time = "2026-07-13T05:22:20.76Z" }, ] [[package]] @@ -24520,7 +24578,7 @@ name = "xmlsec" version = "1.3.17" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "lxml", marker = "python_full_version < '3.13'" }, + { name = "lxml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/14/538b75379e6ab8f688f14d8663e2ab138d9c778bac4999d155b5f33c71c1/xmlsec-1.3.17.tar.gz", hash = "sha256:f3fac9ae679f66585925cc00c5f6839ae36c1d03157619571dee18acc05b9c01", size = 115637, upload-time = "2025-11-11T16:20:46.019Z" } wheels = [ From 060e121e8615355ffc794589980da2e843c1809b Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 7 Jul 2026 22:39:04 +0100 Subject: [PATCH 02/19] Update build system backend --- providers/dq/provider.yaml | 2 +- providers/dq/pyproject.toml | 44 +++++++++---------------------------- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/providers/dq/provider.yaml b/providers/dq/provider.yaml index b886336584a33..6a0d9d4f2ef0d 100644 --- a/providers/dq/provider.yaml +++ b/providers/dq/provider.yaml @@ -29,7 +29,7 @@ description: | state: ready lifecycle: incubation source-date-epoch: 1751587200 -build-system: hatchling +build-system: flit_core # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. diff --git a/providers/dq/pyproject.toml b/providers/dq/pyproject.toml index 1502f9e3d7804..25192edde9bc3 100644 --- a/providers/dq/pyproject.toml +++ b/providers/dq/pyproject.toml @@ -20,15 +20,8 @@ # IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY [build-system] -requires = [ - "hatchling==1.30.1", - "packaging==26.2", - "pathspec==1.1.1", - "pluggy==1.6.0", - "tomli==2.4.1; python_version < '3.11'", - "trove-classifiers==2026.6.1.19", -] -build-backend = "hatchling.build" +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-dq" @@ -81,7 +74,6 @@ dev = [ "apache-airflow-providers-common-compat", "apache-airflow-providers-common-sql", # Additional devel dependencies (do not remove this line and add extra development dependencies) - "apache-airflow-providers-common-sql", ] # To build docs: @@ -121,32 +113,16 @@ apache-airflow-providers-standard = {workspace = true} [project.entry-points."apache_airflow_provider"] provider_info = "airflow.providers.dq.get_provider_info:get_provider_info" -[tool.hatch.version] -path = "src/airflow/providers/dq/__init__.py" +[tool.flit.module] +name = "airflow.providers.dq" -[tool.hatch.build.targets.sdist] +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] include = [ - "docs", - "src/airflow/providers/dq", - "tests", - "NOTICE" -] -exclude = [ - "src/airflow/__init__.py", - "src/airflow/providers/__init__.py", -] - -[tool.hatch.build.targets.custom] -path = "./hatch_build.py" - -artifacts = [ -] - -[tool.hatch.build.targets.wheel] -packages = ['src/airflow'] -artifacts = [ -] -exclude = [ + "docs/", + "provider.yaml", "src/airflow/__init__.py", "src/airflow/providers/__init__.py", + "tests/", ] From d30909728d351ead5c52e60cf6f74e6100d73725 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 7 Jul 2026 22:48:25 +0100 Subject: [PATCH 03/19] Fixup breeze basic tests --- dev/breeze/doc/images/output_build-docs.svg | 16 ++++++++-------- dev/breeze/doc/images/output_build-docs.txt | 2 +- ..._release-management_add-back-references.svg | 16 ++++++++-------- ..._release-management_add-back-references.txt | 2 +- ...se-management_classify-provider-changes.svg | 16 ++++++++-------- ...se-management_classify-provider-changes.txt | 2 +- ...gement_generate-issue-content-providers.svg | 16 ++++++++-------- ...gement_generate-issue-content-providers.txt | 2 +- ...-management_generate-providers-metadata.svg | 18 +++++++++--------- ...-management_generate-providers-metadata.txt | 2 +- ...nagement_prepare-provider-distributions.svg | 16 ++++++++-------- ...nagement_prepare-provider-distributions.txt | 2 +- ...nagement_prepare-provider-documentation.svg | 16 ++++++++-------- ...nagement_prepare-provider-documentation.txt | 2 +- .../output_release-management_publish-docs.svg | 16 ++++++++-------- .../output_release-management_publish-docs.txt | 2 +- ...ut_sbom_generate-providers-requirements.svg | 18 +++++++++--------- ...ut_sbom_generate-providers-requirements.txt | 2 +- .../output_workflow-run_publish-docs.svg | 16 ++++++++-------- .../output_workflow-run_publish-docs.txt | 2 +- dev/breeze/tests/test_selective_checks.py | 2 +- 21 files changed, 93 insertions(+), 93 deletions(-) diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 99bc5dbc0de1e..a0e67180d06c8 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -244,14 +244,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      -weaviate | yandex | ydb | zendesk]...                                                                                  +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  +yandex | ydb | zendesk]...                                                                                             Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 75dd9355bcff2..cd1bfc822ee02 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -e22969fb5e92a3efafb744881fea5ef8 +6b2fb71a678f639237b2501fadb1e610 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 9df4462030484..74a26014c02ea 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -153,14 +153,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      -weaviate | yandex | ydb | zendesk]...                                                                                  +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  +yandex | ydb | zendesk]...                                                                                             Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 152f412a472d3..79f1c82c9c488 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -963d8b3e84cb64aa0d69026638b9fb1b +0f62dc47f54c586441562ca6004ef191 diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index 7e2dae4729d23..fe6774f68dae7 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -165,14 +165,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      -common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch -| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    -jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      -yandex | ydb | zendesk]...                                                                                             +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index c400d2c1da050..dc63186a0cf8d 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -b4c6b4c23a30b7f750a9a8316abd2e91 +20d79ab20c9c8a3642541dae56cc07ca diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 78c943d5732d6..075d604ec1ed0 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -154,14 +154,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      -common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch -| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    -jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      -yandex | ydb | zendesk]...                                                                                             +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 7c5cc1b0fb397..5c011d40cc77d 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -ef90cd93a64627b874f63cca845bddbf +195d4f4d2001bd0b681fa2610264778d diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 4199019d5c56f..c0e373a4c4ced 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -173,15 +173,15 @@ │apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | â”‚ │apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | â”‚ │apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ -│| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | â”‚ -│common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | â”‚ -│elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http â”‚ -│| imap | influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql â”‚ -│| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | â”‚ -│openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | â”‚ -│postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | â”‚ -│singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ -│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ +│| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | â”‚ +│datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol | fab | â”‚ +│facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ +│| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | â”‚ +│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | â”‚ +│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | â”‚ +│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake â”‚ +│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ +│yandex | ydb | zendesk)│ │--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    â”‚ │running metadata generation to only this version of the provider. (TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index bf322a8137ed0..49cc734ebbe27 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -0e161f1be5f2a5b8c6094dcf7859f1c8 +dacac5a22a7ab84d7fdc7e9709375340 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 2a119b59f74a7..f713a828c564c 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -192,14 +192,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      -common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch -| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    -jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      -yandex | ydb | zendesk]...                                                                                             +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index 9cc84f8ed57aa..2b47f1755ef80 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -40982121f81ca247791eee99e22d7214 +b6edff16004d612e687e5f16d01a5b74 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index 4785a550ef45e..74c5c78f5cd46 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -213,14 +213,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      -common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch -| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    -jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      -yandex | ydb | zendesk]...                                                                                             +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index cab1aa0379851..77cd5d480f70b 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -0d1c20ffd24edc17351c5f132a1cc968 +f38ff7c5d51344a214aefe63194a0f71 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index 59c03ad4d408a..b209be1ac5de8 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -192,14 +192,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      -weaviate | yandex | ydb | zendesk]...                                                                                  +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  +yandex | ydb | zendesk]...                                                                                             Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index 585d41d6275cb..466fba39e909b 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -217c089e723c5584da63135bbf011504 +8092a8c5d8b6bc9ecf28c7323f85104f diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index 8e1ee534c9b46..4b243694bf80c 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -187,15 +187,15 @@ │apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | â”‚ │apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | â”‚ │atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | â”‚ -│common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks | â”‚ -│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | â”‚ -│facebook | ftp | git | github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | â”‚ -│informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | â”‚ -│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | â”‚ -│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | â”‚ -│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake â”‚ -│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ -│yandex | ydb | zendesk)│ +│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | â”‚ +│dingding | discord | docker | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git |│ +│github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | jdbc | â”‚ +│jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | â”‚ +│mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle â”‚ +│| pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce â”‚ +│| samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | â”‚ +│standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | â”‚ +│zendesk)│ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported     â”‚ │value to account for the most recent version of the provider (TEXT)│ │--force           Force update providers requirements even if they already exist.│ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index ca3930b91a1a2..8aed44f7ddb9e 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -7cbc3f1d6c0c4f1745702d867f9e7fdc +c07440cc580f3635e601b100ea1aebe6 diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 13e7e9bf14620..42090e1f3cb38 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -207,14 +207,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      -weaviate | yandex | ydb | zendesk]...                                                                                  +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  +yandex | ydb | zendesk]...                                                                                             Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index a86438aee4e27..004f002d222f1 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -cb87f41a19fbc8895cd38aa6351139ae +8ec94bae9a2d0711f3edcf060a79e10d diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 60371125ade90..8b9e2e0057386 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2884,7 +2884,7 @@ def test_upgrade_to_newer_dependencies( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " - "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks elasticsearch " + "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks dq elasticsearch " "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", }, From 97d98e0c7f46bff9b3fcd51a2bce8e14d9de93af Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 7 Jul 2026 23:44:30 +0100 Subject: [PATCH 04/19] Remove unused location --- .../airflow/providers/dq/backends/object_storage.py | 11 +---------- .../dq/tests/unit/dq/backends/test_object_storage.py | 7 +++++++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/providers/dq/src/airflow/providers/dq/backends/object_storage.py b/providers/dq/src/airflow/providers/dq/backends/object_storage.py index c43978a03dbeb..bad5a96267b83 100644 --- a/providers/dq/src/airflow/providers/dq/backends/object_storage.py +++ b/providers/dq/src/airflow/providers/dq/backends/object_storage.py @@ -25,11 +25,8 @@ runs/by_task_instance/dag_id=/task_id=/__.json Latest result for a task-instance page. Last write wins across retries. - rules/by_rule/rule_uid=/__.json - One rule result plus run context: ``{"run": ..., "result": ...}``. - rules/by_task_rule/dag_id=/task_id=/rule_uid=/__.json - Same payload, scoped for task-level rule history views. + One rule result plus run context: ``{"run": ..., "result": ...}``. The duplicate files are intentional read indexes: DQ tasks write once, while the UI reads many times. Keeping these indexes avoids scanning all task runs for common UI views. @@ -155,12 +152,6 @@ def _write_rule_indexes(self, run: DQRun, results: list[RuleResult], timestamp: compact_ts = self._get_safe_key(timestamp) for result in results: payload = {"run": run_context, "result": result.to_dict()} - self._write_rule_index( - self.root / "rules" / "by_rule" / f"rule_uid={result.rule_uid}", - compact_ts, - run.run_uid, - payload, - ) self._write_rule_index( self.root / "rules" diff --git a/providers/dq/tests/unit/dq/backends/test_object_storage.py b/providers/dq/tests/unit/dq/backends/test_object_storage.py index bc6ac2d4b3b28..73cc06294eceb 100644 --- a/providers/dq/tests/unit/dq/backends/test_object_storage.py +++ b/providers/dq/tests/unit/dq/backends/test_object_storage.py @@ -106,6 +106,13 @@ def test_write_run_stores_task_rule_index(self, backend): assert payload["run"]["task_id"] == "dq" assert payload["result"]["rule_uid"] == "rule-1" + def test_write_run_does_not_store_global_rule_index(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = backend.root / "rules" / "by_rule" / "rule_uid=rule-1" + + assert not path.exists() + def test_rule_history_is_newest_first(self, backend): backend.write_run( make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), From 7893b484f44883f6c9e6c0b326e6c120f46f2ac8 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 7 Jul 2026 23:49:31 +0100 Subject: [PATCH 05/19] Fixup tests --- airflow-core/tests/unit/always/test_project_structure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py index 9286bdb5e41a5..af060e888f8d6 100644 --- a/airflow-core/tests/unit/always/test_project_structure.py +++ b/airflow-core/tests/unit/always/test_project_structure.py @@ -107,6 +107,7 @@ def test_providers_modules_should_have_tests(self): "providers/common/compat/tests/unit/common/compat/standard/test_utils.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_base_provider.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_sqs.py", + "providers/dq/tests/unit/dq/test_exceptions.py", "providers/edge3/tests/unit/edge3/cli/test_example_extended_sysinfo.py", "providers/edge3/tests/unit/edge3/models/test_edge_job.py", "providers/edge3/tests/unit/edge3/models/test_edge_logs.py", From e8f1f8c87a81922342cd8e87d90c1c4a3bd240d0 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Wed, 8 Jul 2026 00:49:10 +0100 Subject: [PATCH 06/19] fixup fixup tests --- .github/ISSUE_TEMPLATE/1-airflow_bug_report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 50b2399564e0d..4c010abc43a3b 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -153,6 +153,7 @@ body: - dingding - discord - docker + - dq - edge3 - elasticsearch - exasol From c22ad055bd59a33363462d31a391e9b0e3ce377f Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Wed, 8 Jul 2026 10:36:18 +0100 Subject: [PATCH 07/19] Rename provider --- .../ISSUE_TEMPLATE/1-airflow_bug_report.yml | 2 +- .github/boring-cyborg.yml | 4 +- airflow-core/docs/extra-packages-ref.rst | 2 + .../unit/always/test_project_structure.py | 2 +- dev/breeze/doc/images/output_build-docs.svg | 12 +- dev/breeze/doc/images/output_build-docs.txt | 2 +- ...release-management_add-back-references.svg | 12 +- ...release-management_add-back-references.txt | 2 +- ...e-management_classify-provider-changes.svg | 6 +- ...e-management_classify-provider-changes.txt | 2 +- ...ement_generate-issue-content-providers.svg | 6 +- ...ement_generate-issue-content-providers.txt | 2 +- ...management_generate-providers-metadata.svg | 6 +- ...management_generate-providers-metadata.txt | 2 +- ...agement_prepare-provider-distributions.svg | 6 +- ...agement_prepare-provider-distributions.txt | 2 +- ...agement_prepare-provider-documentation.svg | 6 +- ...agement_prepare-provider-documentation.txt | 2 +- ...output_release-management_publish-docs.svg | 12 +- ...output_release-management_publish-docs.txt | 2 +- ...t_sbom_generate-providers-requirements.svg | 18 +-- ...t_sbom_generate-providers-requirements.txt | 2 +- .../output_workflow-run_publish-docs.svg | 12 +- .../output_workflow-run_publish-docs.txt | 2 +- dev/breeze/tests/test_selective_checks.py | 2 +- docs/spelling_wordlist.txt | 1 + providers/{dq => dataquality}/.gitignore | 0 .../.pre-commit-config.yaml | 8 +- providers/{dq => dataquality}/LICENSE | 0 providers/{dq => dataquality}/NOTICE | 0 providers/{dq => dataquality}/README.rst | 8 +- providers/{dq => dataquality}/docs/agents.rst | 12 +- providers/{dq => dataquality}/docs/assets.rst | 10 +- .../{dq => dataquality}/docs/changelog.rst | 2 +- .../{dq => dataquality}/docs/commits.rst | 4 +- providers/{dq => dataquality}/docs/conf.py | 2 +- .../docs/configurations-ref.rst | 0 .../{dq => dataquality}/docs/decorators.rst | 8 +- providers/{dq => dataquality}/docs/index.rst | 32 +++--- .../installing-providers-from-sources.rst | 0 .../{dq => dataquality}/docs/operators.rst | 20 ++-- providers/{dq => dataquality}/docs/rules.rst | 20 ++-- .../{dq => dataquality}/docs/security.rst | 0 providers/{dq => dataquality}/provider.yaml | 12 +- providers/{dq => dataquality}/pyproject.toml | 14 +-- .../src/airflow/__init__.py | 0 .../src/airflow/providers/__init__.py | 0 .../providers/dataquality}/__init__.py | 4 +- .../airflow/providers/dataquality}/assets.py | 8 +- .../dataquality}/backends/__init__.py | 8 +- .../dataquality}/backends/object_storage.py | 2 +- .../dataquality}/decorators/__init__.py | 0 .../dataquality}/decorators/dq_check.py | 8 +- .../dataquality}/engines/__init__.py | 2 +- .../providers/dataquality}/engines/sql.py | 6 +- .../dataquality}/example_dags/__init__.py | 0 .../example_dq_check_custom_sql.py | 4 +- .../example_dq_check_decorator_dynamic.py | 2 +- .../example_dq_llm_generated_ruleset.py | 10 +- .../example_dq_require_quality.py | 12 +- .../example_dq_ruleset_from_yaml.py | 8 +- .../example_dags/orders_ruleset.yaml | 0 .../providers/dataquality}/exceptions.py | 0 .../dataquality}/get_provider_info.py | 15 ++- .../dataquality}/operators/__init__.py | 2 +- .../dataquality}/operators/dq_check.py | 24 ++-- .../airflow/providers/dataquality}/results.py | 0 .../providers/dataquality}/rules/__init__.py | 4 +- .../providers/dataquality}/rules/checks.py | 0 .../providers/dataquality}/rules/rule.py | 4 +- .../dataquality-rule-authoring}/SKILL.md | 6 +- .../references/ruleset.schema.json | 0 .../providers/dataquality}/version_compat.py | 0 .../{dq => dataquality}/tests/conftest.py | 0 .../tests/system/__init__.py | 0 .../tests/system/dataquality}/__init__.py | 0 .../system/dataquality}/example_dq_check.py | 10 +- .../tests/unit/__init__.py | 0 .../tests/unit/dataquality}/__init__.py | 0 .../unit/dataquality}/backends/__init__.py | 0 .../backends/test_object_storage.py | 12 +- .../unit/dataquality}/decorators/__init__.py | 0 .../dataquality}/decorators/test_dq_check.py | 10 +- .../unit/dataquality}/engines/__init__.py | 0 .../unit/dataquality}/engines/test_sql.py | 4 +- .../unit/dataquality}/operators/__init__.py | 0 .../dataquality}/operators/test_dq_check.py | 16 +-- .../tests/unit/dataquality}/rules/__init__.py | 0 .../unit/dataquality}/rules/test_checks.py | 2 +- .../unit/dataquality}/rules/test_rule.py | 10 +- .../tests/unit/dataquality}/test_assets.py | 6 +- .../tests/unit/dataquality}/test_results.py | 2 +- pyproject.toml | 20 ++-- scripts/ci/docker-compose/remove-sources.yml | 2 +- scripts/ci/docker-compose/tests-sources.yml | 2 +- ...=> generate_dataquality_ruleset_schema.py} | 12 +- uv.lock | 104 +++++++++--------- 97 files changed, 317 insertions(+), 305 deletions(-) rename providers/{dq => dataquality}/.gitignore (100%) rename providers/{dq => dataquality}/.pre-commit-config.yaml (79%) rename providers/{dq => dataquality}/LICENSE (100%) rename providers/{dq => dataquality}/NOTICE (100%) rename providers/{dq => dataquality}/README.rst (89%) rename providers/{dq => dataquality}/docs/agents.rst (80%) rename providers/{dq => dataquality}/docs/assets.rst (83%) rename providers/{dq => dataquality}/docs/changelog.rst (96%) rename providers/{dq => dataquality}/docs/commits.rst (95%) rename providers/{dq => dataquality}/docs/conf.py (98%) rename providers/{dq => dataquality}/docs/configurations-ref.rst (100%) rename providers/{dq => dataquality}/docs/decorators.rst (84%) rename providers/{dq => dataquality}/docs/index.rst (76%) rename providers/{dq => dataquality}/docs/installing-providers-from-sources.rst (100%) rename providers/{dq => dataquality}/docs/operators.rst (82%) rename providers/{dq => dataquality}/docs/rules.rst (90%) rename providers/{dq => dataquality}/docs/security.rst (100%) rename providers/{dq => dataquality}/provider.yaml (89%) rename providers/{dq => dataquality}/pyproject.toml (91%) rename providers/{dq => dataquality}/src/airflow/__init__.py (100%) rename providers/{dq => dataquality}/src/airflow/providers/__init__.py (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/__init__.py (91%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/assets.py (94%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/backends/__init__.py (77%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/backends/object_storage.py (99%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/decorators/__init__.py (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/decorators/dq_check.py (91%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/engines/__init__.py (91%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/engines/sql.py (96%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/example_dags/__init__.py (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/example_dags/example_dq_check_custom_sql.py (96%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/example_dags/example_dq_check_decorator_dynamic.py (98%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/example_dags/example_dq_llm_generated_ruleset.py (93%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/example_dags/example_dq_require_quality.py (88%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/example_dags/example_dq_ruleset_from_yaml.py (90%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/example_dags/orders_ruleset.yaml (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/exceptions.py (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/get_provider_info.py (87%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/operators/__init__.py (91%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/operators/dq_check.py (91%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/results.py (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/rules/__init__.py (88%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/rules/checks.py (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/rules/rule.py (98%) rename providers/{dq/src/airflow/providers/dq/skills/dq-rule-authoring => dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring}/SKILL.md (94%) rename providers/{dq/src/airflow/providers/dq/skills/dq-rule-authoring => dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring}/references/ruleset.schema.json (100%) rename providers/{dq/src/airflow/providers/dq => dataquality/src/airflow/providers/dataquality}/version_compat.py (100%) rename providers/{dq => dataquality}/tests/conftest.py (100%) rename providers/{dq => dataquality}/tests/system/__init__.py (100%) rename providers/{dq/tests/system/dq => dataquality/tests/system/dataquality}/__init__.py (100%) rename providers/{dq/tests/system/dq => dataquality/tests/system/dataquality}/example_dq_check.py (97%) rename providers/{dq => dataquality}/tests/unit/__init__.py (100%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/__init__.py (100%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/backends/__init__.py (100%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/backends/test_object_storage.py (97%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/decorators/__init__.py (100%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/decorators/test_dq_check.py (92%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/engines/__init__.py (100%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/engines/test_sql.py (97%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/operators/__init__.py (100%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/operators/test_dq_check.py (94%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/rules/__init__.py (100%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/rules/test_checks.py (95%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/rules/test_rule.py (96%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/test_assets.py (97%) rename providers/{dq/tests/unit/dq => dataquality/tests/unit/dataquality}/test_results.py (96%) rename scripts/ci/prek/{generate_dq_ruleset_schema.py => generate_dataquality_ruleset_schema.py} (85%) diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 4c010abc43a3b..babfd2b0df5d3 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -149,11 +149,11 @@ body: - common-sql - databricks - datadog + - dataquality - dbt-cloud - dingding - discord - docker - - dq - edge3 - elasticsearch - exasol diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index d874d08dfee7f..22492c7ebf0b4 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -144,8 +144,8 @@ labelPRBasedOnFilePath: provider:docker: - providers/docker/** - provider:dq: - - providers/dq/** + provider:dataquality: + - providers/dataquality/** provider:edge: - providers/edge3/** diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index b6617690de233..4e9b6c95b6400 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -261,6 +261,8 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | datadog | ``pip install 'apache-airflow[datadog]'`` | Datadog hooks and sensors | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ +| dataquality | ``pip install 'apache-airflow[dataquality]'`` | Data Quality hooks and operators | ++---------------------+-----------------------------------------------------+-----------------------------------------------------+ | dbt-cloud | ``pip install 'apache-airflow[dbt-cloud]'`` | dbt Cloud hooks and operators | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | dingding | ``pip install 'apache-airflow[dingding]'`` | Dingding hooks and sensors | diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py index af060e888f8d6..0782f836eb09d 100644 --- a/airflow-core/tests/unit/always/test_project_structure.py +++ b/airflow-core/tests/unit/always/test_project_structure.py @@ -107,7 +107,7 @@ def test_providers_modules_should_have_tests(self): "providers/common/compat/tests/unit/common/compat/standard/test_utils.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_base_provider.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_sqs.py", - "providers/dq/tests/unit/dq/test_exceptions.py", + "providers/dataquality/tests/unit/dataquality/test_exceptions.py", "providers/edge3/tests/unit/edge3/cli/test_example_extended_sysinfo.py", "providers/edge3/tests/unit/edge3/models/test_edge_job.py", "providers/edge3/tests/unit/edge3/models/test_edge_logs.py", diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index a0e67180d06c8..8214bae02bb6b 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -244,12 +244,12 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github -| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality +| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git +| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  yandex | ydb | zendesk]...                                                                                             diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index cd1bfc822ee02..592af20a3509a 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -6b2fb71a678f639237b2501fadb1e610 +650f0de60cf3fdf7767dfd40b91a2675 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 74a26014c02ea..9a44dfb4c583c 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -153,12 +153,12 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github -| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality +| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git +| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  yandex | ydb | zendesk]...                                                                                             diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 79f1c82c9c488..b92168b40a253 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -0f62dc47f54c586441562ca6004ef191 +ff9e85fd0e4ce1969fc862af15407a99 diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index fe6774f68dae7..f02970470843f 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -166,9 +166,9 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    -fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc +| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index dc63186a0cf8d..96dac2cdc561b 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -20d79ab20c9c8a3642541dae56cc07ca +9d3d9f020483dfb9d90b55c2a1d717ee diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 075d604ec1ed0..c4b637d2ab322 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -155,9 +155,9 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    -fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc +| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 5c011d40cc77d..8c7b6cbd28587 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -195d4f4d2001bd0b681fa2610264778d +6f895598495ed23ae78b9a1ca070d041 diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index c0e373a4c4ced..130bb12f325a6 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -174,9 +174,9 @@ │apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | â”‚ │apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ │| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | â”‚ -│datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol | fab | â”‚ -│facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ -│| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | â”‚ +│datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol│ +│| fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | â”‚ +│informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | â”‚ │microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | â”‚ │opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | â”‚ │redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake â”‚ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 49cc734ebbe27..b5a1424d3aa36 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -dacac5a22a7ab84d7fdc7e9709375340 +4df98abc0b2a412910bb4ef2560599fd diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index f713a828c564c..5d1d0e44a2248 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -193,9 +193,9 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    -fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc +| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index 2b47f1755ef80..091e95e9aae8a 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -b6edff16004d612e687e5f16d01a5b74 +7c2e498681b64e4a1357bc4003862c8d diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index 74c5c78f5cd46..a783d6e452406 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -214,9 +214,9 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    -fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc +| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 77cd5d480f70b..7d081d484eae2 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -f38ff7c5d51344a214aefe63194a0f71 +2f2b0ac5ad9d4337ac7ac645a182b2d2 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index b209be1ac5de8..043afef3dfeaf 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -192,12 +192,12 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github -| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality +| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git +| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  yandex | ydb | zendesk]...                                                                                             diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index 466fba39e909b..d2c80bf83297d 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -8092a8c5d8b6bc9ecf28c7323f85104f +794459df3168cc9d0ea3b4fdb12ada1f diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index 4b243694bf80c..0c51c26d64dd4 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -187,15 +187,15 @@ │apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | â”‚ │apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | â”‚ │atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | â”‚ -│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | â”‚ -│dingding | discord | docker | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git |│ -│github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | jdbc | â”‚ -│jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | â”‚ -│mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle â”‚ -│| pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce â”‚ -│| samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | â”‚ -│standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | â”‚ -│zendesk)│ +│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality |│ +│dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp â”‚ +│| git | github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | â”‚ +│jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm│ +│| mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | â”‚ +│oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | â”‚ +│salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite│ +│| ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |│ +│ydb | zendesk)│ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported     â”‚ │value to account for the most recent version of the provider (TEXT)│ │--force           Force update providers requirements even if they already exist.│ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index 8aed44f7ddb9e..c4db876f000f5 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -c07440cc580f3635e601b100ea1aebe6 +74188d0a9923aa0a11de67eca11cca65 diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 42090e1f3cb38..15f8fa38e1233 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -207,12 +207,12 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github -| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality +| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git +| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  yandex | ydb | zendesk]...                                                                                             diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 004f002d222f1..9cbe2fb415404 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -8ec94bae9a2d0711f3edcf060a79e10d +30ce85f34479a082fd77fb47b9557688 diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 8b9e2e0057386..7510add14cd38 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2884,7 +2884,7 @@ def test_upgrade_to_newer_dependencies( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " - "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks dq elasticsearch " + "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks dataquality elasticsearch " "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", }, diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 3382600169700..d4327fad46968 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -294,6 +294,7 @@ conda conf Config config +ConfigDict configfile configMap configmap diff --git a/providers/dq/.gitignore b/providers/dataquality/.gitignore similarity index 100% rename from providers/dq/.gitignore rename to providers/dataquality/.gitignore diff --git a/providers/dq/.pre-commit-config.yaml b/providers/dataquality/.pre-commit-config.yaml similarity index 79% rename from providers/dq/.pre-commit-config.yaml rename to providers/dataquality/.pre-commit-config.yaml index 8aedb6660a14a..5a509f51908f1 100644 --- a/providers/dq/.pre-commit-config.yaml +++ b/providers/dataquality/.pre-commit-config.yaml @@ -24,13 +24,13 @@ default_language_version: repos: - repo: local hooks: - - id: generate-dq-ruleset-schema + - id: generate-dataquality-ruleset-schema name: Generate Data Quality RuleSet schema language: python - entry: ../../scripts/ci/prek/generate_dq_ruleset_schema.py + entry: ../../scripts/ci/prek/generate_dataquality_ruleset_schema.py pass_filenames: false always_run: true files: > (?x) - ^src/airflow/providers/dq/rules/.*\.py$| - ^src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset\.schema\.json$ + ^src/airflow/providers/dataquality/rules/.*\.py$| + ^src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset\.schema\.json$ diff --git a/providers/dq/LICENSE b/providers/dataquality/LICENSE similarity index 100% rename from providers/dq/LICENSE rename to providers/dataquality/LICENSE diff --git a/providers/dq/NOTICE b/providers/dataquality/NOTICE similarity index 100% rename from providers/dq/NOTICE rename to providers/dataquality/NOTICE diff --git a/providers/dq/README.rst b/providers/dataquality/README.rst similarity index 89% rename from providers/dq/README.rst rename to providers/dataquality/README.rst index a6f6ba496bdbc..dd2f97d1a3083 100644 --- a/providers/dq/README.rst +++ b/providers/dataquality/README.rst @@ -15,7 +15,7 @@ specific language governing permissions and limitations under the License. -Package ``apache-airflow-providers-dq`` +Package ``apache-airflow-providers-dataquality`` Release: ``0.1.0`` @@ -28,14 +28,14 @@ storage or local files) so task, run, and rule-level quality can be inspected ov Provider package ---------------- -This package is for the ``dq`` provider. -All classes for this package are included in the ``airflow.providers.dq`` python package. +This package is for the ``dataquality`` provider. +All classes for this package are included in the ``airflow.providers.dataquality`` python package. Installation ------------ You can install this package on top of an existing Airflow installation via -``pip install apache-airflow-providers-dq``. For the minimum Airflow version supported, +``pip install apache-airflow-providers-dataquality``. For the minimum Airflow version supported, see ``Requirements`` below. Requirements diff --git a/providers/dq/docs/agents.rst b/providers/dataquality/docs/agents.rst similarity index 80% rename from providers/dq/docs/agents.rst rename to providers/dataquality/docs/agents.rst index cc820cf88b416..355c07546f632 100644 --- a/providers/dq/docs/agents.rst +++ b/providers/dataquality/docs/agents.rst @@ -20,15 +20,15 @@ Generating rules with an LLM ============================== -Writing a :class:`~airflow.providers.dq.rules.RuleSet` by hand for every table doesn't scale. +Writing a :class:`~airflow.providers.dataquality.rules.RuleSet` by hand for every table doesn't scale. An LLM can propose one from a table's column definitions instead, given the check catalog as context. -The ``dq-rule-authoring`` skill ---------------------------------- +The ``dataquality-rule-authoring`` skill +---------------------------------------- This provider ships an `Agent Skill `__ at -``airflow/providers/dq/skills/dq-rule-authoring/``: a ``SKILL.md`` documenting the +``airflow/providers/dataquality/skills/dataquality-rule-authoring/``: a ``SKILL.md`` documenting the ``RuleSet``/``DQRule`` fields and check catalog, plus a generated JSON Schema (``references/ruleset.schema.json``) for validation. @@ -36,7 +36,7 @@ Point ``common.ai``'s :doc:`AgentSkillsToolset `. .. airflow-providers-commits:: diff --git a/providers/dq/docs/conf.py b/providers/dataquality/docs/conf.py similarity index 98% rename from providers/dq/docs/conf.py rename to providers/dataquality/docs/conf.py index f133bf86729a7..311cddd302c1e 100644 --- a/providers/dq/docs/conf.py +++ b/providers/dataquality/docs/conf.py @@ -22,6 +22,6 @@ import os -os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-dq" +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-dataquality" from docs.provider_conf import * # noqa: F403 diff --git a/providers/dq/docs/configurations-ref.rst b/providers/dataquality/docs/configurations-ref.rst similarity index 100% rename from providers/dq/docs/configurations-ref.rst rename to providers/dataquality/docs/configurations-ref.rst diff --git a/providers/dq/docs/decorators.rst b/providers/dataquality/docs/decorators.rst similarity index 84% rename from providers/dq/docs/decorators.rst rename to providers/dataquality/docs/decorators.rst index 257e0de8bb697..7bd70af01f8b6 100644 --- a/providers/dq/docs/decorators.rst +++ b/providers/dataquality/docs/decorators.rst @@ -20,13 +20,13 @@ ``@task.dq_check`` ===================== -``@task.dq_check`` wraps :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` in +``@task.dq_check`` wraps :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` in the TaskFlow API. ``ruleset`` may be declared as a decorator argument when it exists at Dag-parse time, or returned by the decorated function as a runtime ruleset. ``table`` or ``asset`` are declared as decorator arguments exactly like the plain operator. The decorated function is optional plumbing on top: return ``None`` to run the check exactly as declared. -.. exampleinclude:: /../tests/system/dq/example_dq_check.py +.. exampleinclude:: /../tests/system/dataquality/example_dq_check.py :language: python :dedent: 4 :start-after: [START howto_decorator_dq_check] @@ -35,14 +35,14 @@ function is optional plumbing on top: return ``None`` to run the check exactly a Runtime rule sets -------------------- -Return a :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a YAML path to use a +Return a :class:`~airflow.providers.dataquality.rules.RuleSet`, its dict form, or a YAML path to use a ruleset that is only known at task-execution time -- for example one produced by an upstream task, loaded from a Variable, or generated by an LLM. Return ``None`` to use the ruleset declared on the decorator. Swapping in a different ruleset at execution time: -.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py +.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_check_decorator_dynamic.py :language: python :start-after: [START howto_decorator_dq_check_runtime_ruleset] :end-before: [END howto_decorator_dq_check_runtime_ruleset] diff --git a/providers/dq/docs/index.rst b/providers/dataquality/docs/index.rst similarity index 76% rename from providers/dq/docs/index.rst rename to providers/dataquality/docs/index.rst index 0559b586e2f16..2ebe1f1deb4a9 100644 --- a/providers/dq/docs/index.rst +++ b/providers/dataquality/docs/index.rst @@ -15,8 +15,8 @@ specific language governing permissions and limitations under the License. -``apache-airflow-providers-dq`` -=============================== +``apache-airflow-providers-dataquality`` +======================================== .. toctree:: @@ -45,14 +45,14 @@ :caption: References Configuration - Python API <_api/airflow/providers/dq/index> + Python API <_api/airflow/providers/dataquality/index> .. toctree:: :hidden: :maxdepth: 1 :caption: Resources - PyPI Repository + PyPI Repository Installing from sources .. toctree:: @@ -67,11 +67,11 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/dq/index> + System Tests <_api/tests/system/dataquality/index> -apache-airflow-providers-dq package ------------------------------------ +apache-airflow-providers-dataquality package +-------------------------------------------- ``Data Quality Provider`` @@ -88,14 +88,14 @@ Release: 0.1.0 Provider package ---------------- -This package is for the ``dq`` provider. -All classes for this package are included in the ``airflow.providers.dq`` python package. +This package is for the ``dataquality`` provider. +All classes for this package are included in the ``airflow.providers.dataquality`` python package. Installation ------------ You can install this package on top of an existing Airflow installation via -``pip install apache-airflow-providers-dq``. For the minimum Airflow version supported, +``pip install apache-airflow-providers-dataquality``. For the minimum Airflow version supported, see ``Requirements`` below. Requirements @@ -124,7 +124,7 @@ PIP package Version required Detailed list of commits -apache-airflow-providers-dq package +apache-airflow-providers-dataquality package ------------------------------------------------------ ``Data Quality Provider`` @@ -140,14 +140,14 @@ Release: 0.1.0 Provider package ---------------- -This package is for the ``dq`` provider. -All classes for this package are included in the ``airflow.providers.dq`` python package. +This package is for the ``dataquality`` provider. +All classes for this package are included in the ``airflow.providers.dataquality`` python package. Installation ------------ You can install this package on top of an existing Airflow installation via -``pip install apache-airflow-providers-dq``. +``pip install apache-airflow-providers-dataquality``. For the minimum Airflow version supported, see ``Requirements`` below. Requirements @@ -171,5 +171,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-dq 0.1.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-dq 0.1.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dataquality 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dataquality 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/dq/docs/installing-providers-from-sources.rst b/providers/dataquality/docs/installing-providers-from-sources.rst similarity index 100% rename from providers/dq/docs/installing-providers-from-sources.rst rename to providers/dataquality/docs/installing-providers-from-sources.rst diff --git a/providers/dq/docs/operators.rst b/providers/dataquality/docs/operators.rst similarity index 82% rename from providers/dq/docs/operators.rst rename to providers/dataquality/docs/operators.rst index 70243471e455a..72a357ee4fda6 100644 --- a/providers/dq/docs/operators.rst +++ b/providers/dataquality/docs/operators.rst @@ -20,7 +20,7 @@ ``DQCheckOperator`` ===================== -Use :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` to run a +Use :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` to run a :doc:`ruleset ` against a table and persist per-rule results to the configured results store. Every rule is evaluated and recorded regardless of the task outcome -- a rule failing doesn't stop the others from running, and the full set of results is always written before the @@ -31,7 +31,7 @@ Basic usage Pass a connection, table, and ruleset directly: -.. exampleinclude:: /../tests/system/dq/example_dq_check.py +.. exampleinclude:: /../tests/system/dataquality/example_dq_check.py :language: python :dedent: 4 :start-after: [START howto_operator_dq_check] @@ -40,12 +40,12 @@ Pass a connection, table, and ruleset directly: Parameters ----------- -- ``ruleset`` -- a :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path to a +- ``ruleset`` -- a :class:`~airflow.providers.dataquality.rules.RuleSet`, its dict form, or a path to a YAML ruleset file (see :doc:`rules`). Optional when ``asset`` carries one. - ``table`` -- the table to check. Optional when ``asset`` carries one (falling back to the asset's name). - ``asset`` -- an :class:`~airflow.sdk.Asset` decorated with - :func:`~airflow.providers.dq.assets.asset_quality`. Supplies defaults for ``ruleset``, + :func:`~airflow.providers.dataquality.assets.asset_quality`. Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's outlets so its asset events carry the check summary -- see :doc:`assets`. - ``partition_clause`` -- predicate ANDed into every built-in check's ``WHERE`` clause, e.g. @@ -69,7 +69,7 @@ can't even run. Persisting results -------------------- -Results are persisted to the backend configured under ``[dq] results_path`` (see +Results are persisted to the backend configured under ``[dataquality] results_path`` (see :doc:`configurations-ref`). When that's unset, checks still run and the task still passes or fails normally -- only persisted data quality history is unavailable. There is no per-operator override: every check in a deployment shares one results store, so history stays @@ -79,17 +79,17 @@ Checking an asset -------------------- Attach a ruleset to an :class:`~airflow.sdk.Asset` with -:func:`~airflow.providers.dq.assets.asset_quality`, then pass the asset instead of ``table``/ +:func:`~airflow.providers.dataquality.assets.asset_quality`, then pass the asset instead of ``table``/ ``ruleset``/``conn_id``: -.. exampleinclude:: /../tests/system/dq/example_dq_check.py +.. exampleinclude:: /../tests/system/dataquality/example_dq_check.py :language: python :start-after: [START howto_operator_dq_check_asset] :end-before: [END howto_operator_dq_check_asset] The operator adds the asset to its own outlets automatically, and attaches the check's summary (including its quality ``score``) to the asset event -- which is what makes -:func:`~airflow.providers.dq.assets.require_quality` (see :doc:`assets`) able to gate a +:func:`~airflow.providers.dataquality.assets.require_quality` (see :doc:`assets`) able to gate a downstream consumer Dag on it. ``custom_sql`` checks @@ -98,7 +98,7 @@ downstream consumer Dag on it. Built-in checks are all single-column. For cross-column comparisons, joins, or anything the catalog doesn't cover, use ``custom_sql``: -.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py +.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py :language: python :start-after: [START howto_operator_dq_check_custom_sql] :end-before: [END howto_operator_dq_check_custom_sql] @@ -108,7 +108,7 @@ See :doc:`rules` for the full built-in check catalog and the ``custom_sql`` gram Loading a ruleset from YAML ------------------------------ -.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py +.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py :language: python :start-after: [START howto_operator_dq_check_ruleset_from_yaml] :end-before: [END howto_operator_dq_check_ruleset_from_yaml] diff --git a/providers/dq/docs/rules.rst b/providers/dataquality/docs/rules.rst similarity index 90% rename from providers/dq/docs/rules.rst rename to providers/dataquality/docs/rules.rst index 025ecd4061a24..ac1e76d71ac32 100644 --- a/providers/dq/docs/rules.rst +++ b/providers/dataquality/docs/rules.rst @@ -20,15 +20,15 @@ Rules, rule sets, and checks ============================ -Rules are data, not code. A :class:`~airflow.providers.dq.rules.RuleSet` is a named tuple of -:class:`~airflow.providers.dq.rules.DQRule` -- plain, serializable objects with no behavior of -their own. They describe *what* to check; :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` +Rules are data, not code. A :class:`~airflow.providers.dataquality.rules.RuleSet` is a named tuple of +:class:`~airflow.providers.dataquality.rules.DQRule` -- plain, serializable objects with no behavior of +their own. They describe *what* to check; :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` (see :doc:`operators`) is what actually runs them. Rules are usually written by hand, but they can also be proposed by an LLM -- see :doc:`agents`. .. code-block:: python - from airflow.providers.dq.rules import DQRule, RuleSet + from airflow.providers.dataquality.rules import DQRule, RuleSet orders_ruleset = RuleSet( name="orders_quality", @@ -130,7 +130,7 @@ column-level checks: Conditions ----------- -A :class:`~airflow.providers.dq.rules.Condition` is the pass/fail rule applied to the observed +A :class:`~airflow.providers.dataquality.rules.Condition` is the pass/fail rule applied to the observed value, using the same grammar as the ``common.sql`` check operators: - ``equal_to`` -- exact match. Cannot be combined with any other comparison. @@ -149,7 +149,7 @@ Built-in checks are all single-column. The moment a rule needs to compare two co another table, or use a function the catalog doesn't cover, use ``custom_sql`` -- any SQL statement that resolves to a single scalar, evaluated exactly like a built-in check: -.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py +.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py :language: python :start-after: [START howto_operator_dq_check_custom_sql] :end-before: [END howto_operator_dq_check_custom_sql] @@ -186,14 +186,14 @@ Loading rules from YAML ------------------------- Anywhere a ``RuleSet`` is accepted -- ``DQCheckOperator(ruleset=...)``, -``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.dq.assets.asset_quality` -- a path -string is accepted too, and resolved via :meth:`~airflow.providers.dq.rules.RuleSet.from_file` +``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.dataquality.assets.asset_quality` -- a path +string is accepted too, and resolved via :meth:`~airflow.providers.dataquality.rules.RuleSet.from_file` at Dag-parse time. This keeps rules editable by people who don't write Python. -.. exampleinclude:: /../src/airflow/providers/dq/example_dags/orders_ruleset.yaml +.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/orders_ruleset.yaml :language: yaml -.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py +.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py :language: python :start-after: [START howto_operator_dq_check_ruleset_from_yaml] :end-before: [END howto_operator_dq_check_ruleset_from_yaml] diff --git a/providers/dq/docs/security.rst b/providers/dataquality/docs/security.rst similarity index 100% rename from providers/dq/docs/security.rst rename to providers/dataquality/docs/security.rst diff --git a/providers/dq/provider.yaml b/providers/dataquality/provider.yaml similarity index 89% rename from providers/dq/provider.yaml rename to providers/dataquality/provider.yaml index 6a0d9d4f2ef0d..c788e673fd8de 100644 --- a/providers/dq/provider.yaml +++ b/providers/dataquality/provider.yaml @@ -16,7 +16,7 @@ # under the License. --- -package-name: apache-airflow-providers-dq +package-name: apache-airflow-providers-dataquality name: Data Quality description: | ``Data Quality Provider`` @@ -40,22 +40,22 @@ versions: integrations: - integration-name: Data Quality - external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-dq/ + external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-dataquality/ how-to-guide: - - /docs/apache-airflow-providers-dq/operators.rst + - /docs/apache-airflow-providers-dataquality/operators.rst tags: [software] operators: - integration-name: Data Quality python-modules: - - airflow.providers.dq.operators.dq_check + - airflow.providers.dataquality.operators.dq_check task-decorators: - - class-name: airflow.providers.dq.decorators.dq_check.dq_check_task + - class-name: airflow.providers.dataquality.decorators.dq_check.dq_check_task name: dq_check config: - dq: + dataquality: description: | Configuration for the Data Quality provider results store. options: diff --git a/providers/dq/pyproject.toml b/providers/dataquality/pyproject.toml similarity index 91% rename from providers/dq/pyproject.toml rename to providers/dataquality/pyproject.toml index 25192edde9bc3..10cd4887598f5 100644 --- a/providers/dq/pyproject.toml +++ b/providers/dataquality/pyproject.toml @@ -24,9 +24,9 @@ requires = ["flit_core==3.12.0"] build-backend = "flit_core.buildapi" [project] -name = "apache-airflow-providers-dq" +name = "apache-airflow-providers-dataquality" version = "0.1.0" -description = "Provider package apache-airflow-providers-dq for Apache Airflow" +description = "Provider package apache-airflow-providers-dataquality for Apache Airflow" readme = "README.rst" license = "Apache-2.0" license-files = ['LICENSE', 'NOTICE'] @@ -36,7 +36,7 @@ authors = [ maintainers = [ {name="Apache Software Foundation", email="dev@airflow.apache.org"}, ] -keywords = [ "airflow-provider", "dq", "airflow", "integration" ] +keywords = [ "airflow-provider", "dataquality", "airflow", "integration" ] classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -102,8 +102,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dq/0.1.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dq/0.1.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dataquality/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dataquality/0.1.0/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" @@ -111,10 +111,10 @@ apache-airflow-providers-standard = {workspace = true} "YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" [project.entry-points."apache_airflow_provider"] -provider_info = "airflow.providers.dq.get_provider_info:get_provider_info" +provider_info = "airflow.providers.dataquality.get_provider_info:get_provider_info" [tool.flit.module] -name = "airflow.providers.dq" +name = "airflow.providers.dataquality" # Explicit sdist contents so the build does not rely on VCS information # (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). diff --git a/providers/dq/src/airflow/__init__.py b/providers/dataquality/src/airflow/__init__.py similarity index 100% rename from providers/dq/src/airflow/__init__.py rename to providers/dataquality/src/airflow/__init__.py diff --git a/providers/dq/src/airflow/providers/__init__.py b/providers/dataquality/src/airflow/providers/__init__.py similarity index 100% rename from providers/dq/src/airflow/providers/__init__.py rename to providers/dataquality/src/airflow/providers/__init__.py diff --git a/providers/dq/src/airflow/providers/dq/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/__init__.py similarity index 91% rename from providers/dq/src/airflow/providers/dq/__init__.py rename to providers/dataquality/src/airflow/providers/dataquality/__init__.py index 4fd31dec8bc1a..57e75da020e0b 100644 --- a/providers/dq/src/airflow/providers/dq/__init__.py +++ b/providers/dataquality/src/airflow/providers/dataquality/__init__.py @@ -34,4 +34,6 @@ if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "3.0.0" ): - raise RuntimeError(f"The package `apache-airflow-providers-dq:{__version__}` needs Apache Airflow 3.0.0+") + raise RuntimeError( + f"The package `apache-airflow-providers-dataquality:{__version__}` needs Apache Airflow 3.0.0+" + ) diff --git a/providers/dq/src/airflow/providers/dq/assets.py b/providers/dataquality/src/airflow/providers/dataquality/assets.py similarity index 94% rename from providers/dq/src/airflow/providers/dq/assets.py rename to providers/dataquality/src/airflow/providers/dataquality/assets.py index 513587a68d9b4..46020a5d23a6a 100644 --- a/providers/dq/src/airflow/providers/dq/assets.py +++ b/providers/dataquality/src/airflow/providers/dataquality/assets.py @@ -27,8 +27,8 @@ import logging from typing import TYPE_CHECKING, Any -from airflow.providers.dq.exceptions import DQRuleValidationError -from airflow.providers.dq.rules import RuleSet +from airflow.providers.dataquality.exceptions import DQRuleValidationError +from airflow.providers.dataquality.rules import RuleSet from airflow.sdk import task if TYPE_CHECKING: @@ -54,7 +54,7 @@ def asset_quality( Attach data quality configuration to an asset, returning the same asset. :param asset: The asset the rules describe. - :param ruleset: A :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path + :param ruleset: A :class:`~airflow.providers.dataquality.rules.RuleSet`, its dict form, or a path to a YAML ruleset file (resolved eagerly, at Dag-parse time). :param conn_id: Default connection a check operator should use for this asset. :param table: Default table to check; falls back to the asset name when unset. @@ -144,7 +144,7 @@ def require_quality( """ Gate a Dag run on the data quality score attached to one of its triggering asset events. - Reads the summary a :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` + Reads the summary a :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dq.result"]`` (see :func:`asset_quality`), and short-circuits the run — skipping every downstream task — when that summary is missing or its score is below ``min_score``. Call it inside a Dag diff --git a/providers/dq/src/airflow/providers/dq/backends/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/backends/__init__.py similarity index 77% rename from providers/dq/src/airflow/providers/dq/backends/__init__.py rename to providers/dataquality/src/airflow/providers/dataquality/backends/__init__.py index 7fe7bade843a5..58e7df4024d93 100644 --- a/providers/dq/src/airflow/providers/dq/backends/__init__.py +++ b/providers/dataquality/src/airflow/providers/dataquality/backends/__init__.py @@ -16,17 +16,17 @@ # under the License. from __future__ import annotations -from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend __all__ = ["ObjectStorageResultsBackend", "get_backend_from_config"] def get_backend_from_config() -> ObjectStorageResultsBackend | None: - """Build the results backend configured under ``[dq]``, or ``None`` when not configured.""" + """Build the results backend configured under ``[dataquality]``, or ``None`` when not configured.""" from airflow.providers.common.compat.sdk import conf - results_path = conf.get("dq", "results_path", fallback=None) + results_path = conf.get("dataquality", "results_path", fallback=None) if not results_path: return None - conn_id = conf.get("dq", "results_conn_id", fallback=None) + conn_id = conf.get("dataquality", "results_conn_id", fallback=None) return ObjectStorageResultsBackend(results_path=results_path, conn_id=conn_id or None) diff --git a/providers/dq/src/airflow/providers/dq/backends/object_storage.py b/providers/dataquality/src/airflow/providers/dataquality/backends/object_storage.py similarity index 99% rename from providers/dq/src/airflow/providers/dq/backends/object_storage.py rename to providers/dataquality/src/airflow/providers/dataquality/backends/object_storage.py index bad5a96267b83..94d0236f975db 100644 --- a/providers/dq/src/airflow/providers/dq/backends/object_storage.py +++ b/providers/dataquality/src/airflow/providers/dataquality/backends/object_storage.py @@ -39,7 +39,7 @@ from datetime import datetime, timezone from typing import Any -from airflow.providers.dq.results import DQRun, RuleResult, build_summary +from airflow.providers.dataquality.results import DQRun, RuleResult, build_summary from airflow.sdk import ObjectStoragePath log = logging.getLogger(__name__) diff --git a/providers/dq/src/airflow/providers/dq/decorators/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/decorators/__init__.py similarity index 100% rename from providers/dq/src/airflow/providers/dq/decorators/__init__.py rename to providers/dataquality/src/airflow/providers/dataquality/decorators/__init__.py diff --git a/providers/dq/src/airflow/providers/dq/decorators/dq_check.py b/providers/dataquality/src/airflow/providers/dataquality/decorators/dq_check.py similarity index 91% rename from providers/dq/src/airflow/providers/dq/decorators/dq_check.py rename to providers/dataquality/src/airflow/providers/dataquality/decorators/dq_check.py index 0385187b1d39d..2da6bf423717a 100644 --- a/providers/dq/src/airflow/providers/dq/decorators/dq_check.py +++ b/providers/dataquality/src/airflow/providers/dataquality/decorators/dq_check.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""TaskFlow decorator for :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator`.""" +"""TaskFlow decorator for :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator`.""" from __future__ import annotations @@ -27,8 +27,8 @@ determine_kwargs, task_decorator_factory, ) -from airflow.providers.dq.operators.dq_check import DQCheckOperator -from airflow.providers.dq.rules import RuleSet +from airflow.providers.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.dataquality.rules import RuleSet from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION if TYPE_CHECKING: @@ -89,7 +89,7 @@ def dq_check_task( ``ruleset`` may be passed as a decorator argument when known at Dag-parse time, or returned by the decorated function at runtime. ``table`` or ``asset`` are passed through ``kwargs`` - exactly as on :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator`. The + exactly as on :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator`. The function itself is optional plumbing: return ``None`` to run the check as declared, or a ruleset to use for this run. diff --git a/providers/dq/src/airflow/providers/dq/engines/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/engines/__init__.py similarity index 91% rename from providers/dq/src/airflow/providers/dq/engines/__init__.py rename to providers/dataquality/src/airflow/providers/dataquality/engines/__init__.py index 98a6f6fe9ba80..70d25099b75cd 100644 --- a/providers/dq/src/airflow/providers/dq/engines/__init__.py +++ b/providers/dataquality/src/airflow/providers/dataquality/engines/__init__.py @@ -16,6 +16,6 @@ # under the License. from __future__ import annotations -from airflow.providers.dq.engines.sql import Observation, SQLDQEngine +from airflow.providers.dataquality.engines.sql import Observation, SQLDQEngine __all__ = ["Observation", "SQLDQEngine"] diff --git a/providers/dq/src/airflow/providers/dq/engines/sql.py b/providers/dataquality/src/airflow/providers/dataquality/engines/sql.py similarity index 96% rename from providers/dq/src/airflow/providers/dq/engines/sql.py rename to providers/dataquality/src/airflow/providers/dataquality/engines/sql.py index 43c2f9579f8df..ed21549eee450 100644 --- a/providers/dq/src/airflow/providers/dq/engines/sql.py +++ b/providers/dataquality/src/airflow/providers/dataquality/engines/sql.py @@ -23,12 +23,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from airflow.providers.dq.rules import CUSTOM_SQL_CHECK -from airflow.providers.dq.rules.checks import CHECK_SPECS +from airflow.providers.dataquality.rules import CUSTOM_SQL_CHECK +from airflow.providers.dataquality.rules.checks import CHECK_SPECS if TYPE_CHECKING: from airflow.providers.common.sql.hooks.sql import DbApiHook - from airflow.providers.dq.rules import DQRule, RuleSet + from airflow.providers.dataquality.rules import DQRule, RuleSet log = logging.getLogger(__name__) diff --git a/providers/dq/src/airflow/providers/dq/example_dags/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/example_dags/__init__.py similarity index 100% rename from providers/dq/src/airflow/providers/dq/example_dags/__init__.py rename to providers/dataquality/src/airflow/providers/dataquality/example_dags/__init__.py diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py similarity index 96% rename from providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py rename to providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py index f07f746be9849..c48b39b8611c7 100644 --- a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py +++ b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py @@ -31,8 +31,8 @@ from pathlib import Path from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dq.operators.dq_check import DQCheckOperator -from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.providers.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.dataquality.rules import DQRule, RuleSet, Severity from airflow.sdk import DAG DAG_ID = "example_dq_check_custom_sql" diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_decorator_dynamic.py similarity index 98% rename from providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py rename to providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_decorator_dynamic.py index 2a4b782539a11..542875a64cb36 100644 --- a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py +++ b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_decorator_dynamic.py @@ -30,7 +30,7 @@ from pathlib import Path from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.providers.dataquality.rules import DQRule, RuleSet from airflow.sdk import DAG, task DAG_ID = "example_dq_check_decorator_dynamic" diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_llm_generated_ruleset.py similarity index 93% rename from providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py rename to providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_llm_generated_ruleset.py index 7ecfd743ceb91..c3a8ea8f767ab 100644 --- a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py +++ b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_llm_generated_ruleset.py @@ -15,13 +15,13 @@ # specific language governing permissions and limitations # under the License. """ -Example DAG: generate a ``RuleSet`` with an LLM, guided by the ``dq-rule-authoring`` skill. +Example DAG: generate a ``RuleSet`` with an LLM, guided by the ``dataquality-rule-authoring`` skill. Requires the optional ``apache-airflow-providers-common-ai[skills]`` package and a configured LLM connection (``llm_conn_id``); this Dag is not registered when common.ai isn't installed. An LLM is asked to propose data quality rules for a table's columns. Its ``system_prompt`` -points it at the ``dq-rule-authoring`` skill shipped with this provider (via +points it at the ``dataquality-rule-authoring`` skill shipped with this provider (via ``AgentSkillsToolset``), so it knows the exact ``RuleSet``/``DQRule`` schema, the built-in check catalog, and the ``Condition`` grammar, instead of guessing at field or check names. ``output_type=RuleSet`` makes pydantic-ai validate -- and self-correct -- the model's output @@ -39,7 +39,7 @@ from pathlib import Path from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dq.rules import RuleSet +from airflow.providers.dataquality.rules import RuleSet from airflow.sdk import DAG, task try: @@ -53,7 +53,7 @@ RESULTS_PATH = Path("/tmp/airflow_dq_example/results") # The skill ships next to this Dag's package, not next to this file -- resolve relative to # __file__ so the path holds regardless of the Dag processor's working directory. -SKILLS_DIR = Path(__file__).parent.parent / "skills" / "dq-rule-authoring" +SKILLS_DIR = Path(__file__).parent.parent / "skills" / "dataquality-rule-authoring" os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") @@ -98,7 +98,7 @@ llm_conn_id="pydanticai_default", system_prompt=( "You are a data quality engineer. Before answering, consult the " - "dq-rule-authoring skill for the exact RuleSet/DQRule schema, the built-in " + "dataquality-rule-authoring skill for the exact RuleSet/DQRule schema, the built-in " "check catalog, and the Condition grammar -- do not invent field or check names." ), output_type=RuleSet, diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_require_quality.py similarity index 88% rename from providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py rename to providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_require_quality.py index 124156604a11e..539940c422172 100644 --- a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py +++ b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_require_quality.py @@ -20,11 +20,11 @@ Two Dags, wired together only through the ``dq_gated_orders`` asset: - ``example_dq_require_quality_producer`` checks a table and produces the asset. - :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` attaches its summary + :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` attaches its summary (including the quality ``score``) to the asset event automatically because the asset carries - quality config attached with :func:`~airflow.providers.dq.assets.asset_quality`. + quality config attached with :func:`~airflow.providers.dataquality.assets.asset_quality`. - ``example_dq_require_quality_consumer`` is scheduled by that asset. Its first task, built by - :func:`~airflow.providers.dq.assets.require_quality`, reads the score off the triggering + :func:`~airflow.providers.dataquality.assets.require_quality`, reads the score off the triggering asset event and short-circuits -- skipping every downstream task -- when it's missing or below ``min_score``. Downstream tasks never see a run triggered by a bad batch of data. """ @@ -36,9 +36,9 @@ from pathlib import Path from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dq.assets import asset_quality, require_quality -from airflow.providers.dq.operators.dq_check import DQCheckOperator -from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.providers.dataquality.assets import asset_quality, require_quality +from airflow.providers.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.dataquality.rules import DQRule, RuleSet from airflow.sdk import DAG, Asset, task CONN_ID = "sqlite_default" diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py similarity index 90% rename from providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py rename to providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py index 25b9a6cf76abd..077f44e67ae2d 100644 --- a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py +++ b/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py @@ -17,10 +17,10 @@ """ Example DAG: load a ruleset from a YAML file instead of declaring it in Python. -A path string is accepted anywhere a :class:`~airflow.providers.dq.rules.RuleSet` is -- +A path string is accepted anywhere a :class:`~airflow.providers.dataquality.rules.RuleSet` is -- ``DQCheckOperator(ruleset=...)``, ``@task.dq_check(ruleset=...)``, and -:func:`~airflow.providers.dq.assets.asset_quality` all resolve it via -:meth:`~airflow.providers.dq.rules.RuleSet.from_file` at Dag-parse time. This keeps rules +:func:`~airflow.providers.dataquality.assets.asset_quality` all resolve it via +:meth:`~airflow.providers.dataquality.rules.RuleSet.from_file` at Dag-parse time. This keeps rules editable by people who don't write Python -- a data steward can change ``orders_ruleset.yaml`` without touching the Dag file. """ @@ -32,7 +32,7 @@ from pathlib import Path from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dataquality.operators.dq_check import DQCheckOperator from airflow.sdk import DAG DAG_ID = "example_dq_ruleset_from_yaml" diff --git a/providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml b/providers/dataquality/src/airflow/providers/dataquality/example_dags/orders_ruleset.yaml similarity index 100% rename from providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml rename to providers/dataquality/src/airflow/providers/dataquality/example_dags/orders_ruleset.yaml diff --git a/providers/dq/src/airflow/providers/dq/exceptions.py b/providers/dataquality/src/airflow/providers/dataquality/exceptions.py similarity index 100% rename from providers/dq/src/airflow/providers/dq/exceptions.py rename to providers/dataquality/src/airflow/providers/dataquality/exceptions.py diff --git a/providers/dq/src/airflow/providers/dq/get_provider_info.py b/providers/dataquality/src/airflow/providers/dataquality/get_provider_info.py similarity index 87% rename from providers/dq/src/airflow/providers/dq/get_provider_info.py rename to providers/dataquality/src/airflow/providers/dataquality/get_provider_info.py index fea5885430a63..2f945b1e286db 100644 --- a/providers/dq/src/airflow/providers/dq/get_provider_info.py +++ b/providers/dataquality/src/airflow/providers/dataquality/get_provider_info.py @@ -23,28 +23,31 @@ def get_provider_info(): return { - "package-name": "apache-airflow-providers-dq", + "package-name": "apache-airflow-providers-dataquality", "name": "Data Quality", "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", "integrations": [ { "integration-name": "Data Quality", - "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-dq/", - "how-to-guide": ["/docs/apache-airflow-providers-dq/operators.rst"], + "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-dataquality/", + "how-to-guide": ["/docs/apache-airflow-providers-dataquality/operators.rst"], "tags": ["software"], } ], "operators": [ { "integration-name": "Data Quality", - "python-modules": ["airflow.providers.dq.operators.dq_check"], + "python-modules": ["airflow.providers.dataquality.operators.dq_check"], } ], "task-decorators": [ - {"class-name": "airflow.providers.dq.decorators.dq_check.dq_check_task", "name": "dq_check"} + { + "class-name": "airflow.providers.dataquality.decorators.dq_check.dq_check_task", + "name": "dq_check", + } ], "config": { - "dq": { + "dataquality": { "description": "Configuration for the Data Quality provider results store.\n", "options": { "results_path": { diff --git a/providers/dq/src/airflow/providers/dq/operators/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/operators/__init__.py similarity index 91% rename from providers/dq/src/airflow/providers/dq/operators/__init__.py rename to providers/dataquality/src/airflow/providers/dataquality/operators/__init__.py index f1734cd8846f9..73c89bc03920f 100644 --- a/providers/dq/src/airflow/providers/dq/operators/__init__.py +++ b/providers/dataquality/src/airflow/providers/dataquality/operators/__init__.py @@ -16,6 +16,6 @@ # under the License. from __future__ import annotations -from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dataquality.operators.dq_check import DQCheckOperator __all__ = ["DQCheckOperator"] diff --git a/providers/dq/src/airflow/providers/dq/operators/dq_check.py b/providers/dataquality/src/airflow/providers/dataquality/operators/dq_check.py similarity index 91% rename from providers/dq/src/airflow/providers/dq/operators/dq_check.py rename to providers/dataquality/src/airflow/providers/dataquality/operators/dq_check.py index f006a54686010..fb6cc52a281d2 100644 --- a/providers/dq/src/airflow/providers/dq/operators/dq_check.py +++ b/providers/dataquality/src/airflow/providers/dataquality/operators/dq_check.py @@ -22,18 +22,18 @@ from typing import TYPE_CHECKING, Any, cast from airflow.providers.common.sql.operators.sql import BaseSQLOperator -from airflow.providers.dq.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config -from airflow.providers.dq.backends import get_backend_from_config -from airflow.providers.dq.engines.sql import SQLDQEngine -from airflow.providers.dq.exceptions import DQCheckFailedError -from airflow.providers.dq.results import ERROR, FAIL, PASS, WARN, DQRun, RuleResult, build_summary -from airflow.providers.dq.rules import RuleSet, describe_rule +from airflow.providers.dataquality.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config +from airflow.providers.dataquality.backends import get_backend_from_config +from airflow.providers.dataquality.engines.sql import SQLDQEngine +from airflow.providers.dataquality.exceptions import DQCheckFailedError +from airflow.providers.dataquality.results import ERROR, FAIL, PASS, WARN, DQRun, RuleResult, build_summary +from airflow.providers.dataquality.rules import RuleSet, describe_rule from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION if TYPE_CHECKING: from airflow.providers.common.sql.hooks.sql import DbApiHook - from airflow.providers.dq.engines.sql import Observation - from airflow.providers.dq.rules.rule import Condition, Dimension + from airflow.providers.dataquality.engines.sql import Observation + from airflow.providers.dataquality.rules.rule import Condition, Dimension from airflow.sdk import Asset, Context log = logging.getLogger(__name__) @@ -49,11 +49,11 @@ class DQCheckOperator(BaseSQLOperator): (a check query failing) always fail the task; rule failures fail the task according to ``fail_on``. - :param ruleset: A :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path + :param ruleset: A :class:`~airflow.providers.dataquality.rules.RuleSet`, its dict form, or a path to a YAML ruleset file. Optional when ``asset`` carries one. :param table: The table to run checks against. Optional when ``asset`` carries one (falling back to the asset name). - :param asset: An asset decorated with :func:`~airflow.providers.dq.assets.asset_quality`. + :param asset: An asset decorated with :func:`~airflow.providers.dataquality.assets.asset_quality`. Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's outlets so its asset events carry the check summary. @@ -65,7 +65,7 @@ class DQCheckOperator(BaseSQLOperator): :param conn_id: Connection to the database to check (any ``DbApiHook``-compatible type). :param database: Optional database/schema overriding the connection's default. - Results are persisted to the backend configured under ``[dq] results_path``; when that's + Results are persisted to the backend configured under ``[dataquality] results_path``; when that's unset, checks still run but no history is persisted. There is no per-operator override -- all checks in a deployment share one results store. """ @@ -197,7 +197,7 @@ def _outlet_asset_names(self) -> list[str]: def _persist(self, run: DQRun, results: list[RuleResult]) -> None: backend = get_backend_from_config() if backend is None: - self.log.info("No [dq] results_path configured; skipping results persistence") + self.log.info("No [dataquality] results_path configured; skipping results persistence") return # Persistence is best-effort: an unreachable results store leaves a gap in # history but must not change the outcome of the check itself. diff --git a/providers/dq/src/airflow/providers/dq/results.py b/providers/dataquality/src/airflow/providers/dataquality/results.py similarity index 100% rename from providers/dq/src/airflow/providers/dq/results.py rename to providers/dataquality/src/airflow/providers/dataquality/results.py diff --git a/providers/dq/src/airflow/providers/dq/rules/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/rules/__init__.py similarity index 88% rename from providers/dq/src/airflow/providers/dq/rules/__init__.py rename to providers/dataquality/src/airflow/providers/dataquality/rules/__init__.py index 1536086480383..208dc2d83da2e 100644 --- a/providers/dq/src/airflow/providers/dq/rules/__init__.py +++ b/providers/dataquality/src/airflow/providers/dataquality/rules/__init__.py @@ -16,8 +16,8 @@ # under the License. from __future__ import annotations -from airflow.providers.dq.rules.checks import CHECK_SPECS, CheckSpec, Dimension -from airflow.providers.dq.rules.rule import ( +from airflow.providers.dataquality.rules.checks import CHECK_SPECS, CheckSpec, Dimension +from airflow.providers.dataquality.rules.rule import ( CUSTOM_SQL_CHECK, Condition, DQRule, diff --git a/providers/dq/src/airflow/providers/dq/rules/checks.py b/providers/dataquality/src/airflow/providers/dataquality/rules/checks.py similarity index 100% rename from providers/dq/src/airflow/providers/dq/rules/checks.py rename to providers/dataquality/src/airflow/providers/dataquality/rules/checks.py diff --git a/providers/dq/src/airflow/providers/dq/rules/rule.py b/providers/dataquality/src/airflow/providers/dataquality/rules/rule.py similarity index 98% rename from providers/dq/src/airflow/providers/dq/rules/rule.py rename to providers/dataquality/src/airflow/providers/dataquality/rules/rule.py index 8151be8e41143..13d5fbf360870 100644 --- a/providers/dq/src/airflow/providers/dq/rules/rule.py +++ b/providers/dataquality/src/airflow/providers/dataquality/rules/rule.py @@ -26,8 +26,8 @@ import yaml from pydantic import BaseModel, ConfigDict, Field, model_validator -from airflow.providers.dq.exceptions import DQRuleValidationError -from airflow.providers.dq.rules.checks import CHECK_SPECS, Dimension +from airflow.providers.dataquality.exceptions import DQRuleValidationError +from airflow.providers.dataquality.rules.checks import CHECK_SPECS, Dimension CUSTOM_SQL_CHECK = "custom_sql" diff --git a/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md b/providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/SKILL.md similarity index 94% rename from providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md rename to providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/SKILL.md index f7a7693d49e41..97dc9762b7fa2 100644 --- a/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md +++ b/providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/SKILL.md @@ -1,12 +1,12 @@ --- -name: dq-rule-authoring -description: "Generate a RuleSet/DQRule JSON payload for Apache Airflow's dq provider from a table's column definitions. Use this whenever asked to write, generate, or suggest data quality rules, checks, or a ruleset for a table or dataset -- especially when the output is structured JSON handed to DQCheckOperator, @task.dq_check, asset_quality(), or RuleSet.from_file()." +name: dataquality-rule-authoring +description: "Generate a RuleSet/DQRule JSON payload for Apache Airflow's dataquality provider from a table's column definitions. Use this whenever asked to write, generate, or suggest data quality rules, checks, or a ruleset for a table or dataset -- especially when the output is structured JSON handed to DQCheckOperator, @task.dq_check, asset_quality(), or RuleSet.from_file()." --- -# dq rule authoring +# dataquality rule authoring You are producing a **RuleSet**: a JSON object naming a table's data quality checks. This document is the schema and the full list of valid values -- do not guess field names or check diff --git a/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json b/providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json similarity index 100% rename from providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json rename to providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json diff --git a/providers/dq/src/airflow/providers/dq/version_compat.py b/providers/dataquality/src/airflow/providers/dataquality/version_compat.py similarity index 100% rename from providers/dq/src/airflow/providers/dq/version_compat.py rename to providers/dataquality/src/airflow/providers/dataquality/version_compat.py diff --git a/providers/dq/tests/conftest.py b/providers/dataquality/tests/conftest.py similarity index 100% rename from providers/dq/tests/conftest.py rename to providers/dataquality/tests/conftest.py diff --git a/providers/dq/tests/system/__init__.py b/providers/dataquality/tests/system/__init__.py similarity index 100% rename from providers/dq/tests/system/__init__.py rename to providers/dataquality/tests/system/__init__.py diff --git a/providers/dq/tests/system/dq/__init__.py b/providers/dataquality/tests/system/dataquality/__init__.py similarity index 100% rename from providers/dq/tests/system/dq/__init__.py rename to providers/dataquality/tests/system/dataquality/__init__.py diff --git a/providers/dq/tests/system/dq/example_dq_check.py b/providers/dataquality/tests/system/dataquality/example_dq_check.py similarity index 97% rename from providers/dq/tests/system/dq/example_dq_check.py rename to providers/dataquality/tests/system/dataquality/example_dq_check.py index 62322478145b4..16a58f2cf3325 100644 --- a/providers/dq/tests/system/dq/example_dq_check.py +++ b/providers/dataquality/tests/system/dataquality/example_dq_check.py @@ -18,7 +18,7 @@ Example DAG for the Data Quality provider's ``DQCheckOperator``. Runs against the ``sqlite_default`` connection (Airflow's default local backend, no extra -infrastructure required). Results are persisted through the ``[dq] results_path`` config +infrastructure required). Results are persisted through the ``[dataquality] results_path`` config option, seeded here via ``AIRFLOW__DQ__RESULTS_PATH`` so the history can be inspected afterwards under ``/tmp/airflow_dq_example/results`` -- a real deployment would instead set ``results_path`` in ``airflow.cfg`` (or its env var) once, for every check to share. @@ -55,9 +55,9 @@ from pathlib import Path from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dq.assets import asset_quality -from airflow.providers.dq.operators.dq_check import DQCheckOperator -from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.providers.dataquality.assets import asset_quality +from airflow.providers.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.dataquality.rules import DQRule, RuleSet, Severity from airflow.sdk import DAG, Asset, task DAG_ID = "example_dq_check" @@ -66,7 +66,7 @@ RESULTS_PATH = Path("/tmp/airflow_dq_example/results") # DQCheckOperator has no per-operator results-store override; every check in a deployment -# shares the one store configured under [dq] results_path. setdefault() so a real deployment's +# shares the one store configured under [dataquality] results_path. setdefault() so a real deployment's # own config is never overridden. os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") diff --git a/providers/dq/tests/unit/__init__.py b/providers/dataquality/tests/unit/__init__.py similarity index 100% rename from providers/dq/tests/unit/__init__.py rename to providers/dataquality/tests/unit/__init__.py diff --git a/providers/dq/tests/unit/dq/__init__.py b/providers/dataquality/tests/unit/dataquality/__init__.py similarity index 100% rename from providers/dq/tests/unit/dq/__init__.py rename to providers/dataquality/tests/unit/dataquality/__init__.py diff --git a/providers/dq/tests/unit/dq/backends/__init__.py b/providers/dataquality/tests/unit/dataquality/backends/__init__.py similarity index 100% rename from providers/dq/tests/unit/dq/backends/__init__.py rename to providers/dataquality/tests/unit/dataquality/backends/__init__.py diff --git a/providers/dq/tests/unit/dq/backends/test_object_storage.py b/providers/dataquality/tests/unit/dataquality/backends/test_object_storage.py similarity index 97% rename from providers/dq/tests/unit/dq/backends/test_object_storage.py rename to providers/dataquality/tests/unit/dataquality/backends/test_object_storage.py index 73cc06294eceb..88187f5bd2100 100644 --- a/providers/dq/tests/unit/dq/backends/test_object_storage.py +++ b/providers/dataquality/tests/unit/dataquality/backends/test_object_storage.py @@ -21,10 +21,10 @@ import pytest -from airflow.providers.dq.backends import get_backend_from_config -from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend -from airflow.providers.dq.results import DQRun, RuleResult -from airflow.providers.dq.rules import Severity +from airflow.providers.dataquality.backends import get_backend_from_config +from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dataquality.results import DQRun, RuleResult +from airflow.providers.dataquality.rules import Severity from tests_common.test_utils.config import conf_vars @@ -354,11 +354,11 @@ def test_read_task_rule_history_cursor_keeps_same_timestamp_records(self, backen class TestGetBackendFromConfig: - @conf_vars({("dq", "results_path"): None}) + @conf_vars({("dataquality", "results_path"): None}) def test_no_results_path_returns_none(self): assert get_backend_from_config() is None def test_results_path_builds_object_storage_backend(self, tmp_path): - with conf_vars({("dq", "results_path"): f"file://{tmp_path}"}): + with conf_vars({("dataquality", "results_path"): f"file://{tmp_path}"}): backend = get_backend_from_config() assert isinstance(backend, ObjectStorageResultsBackend) diff --git a/providers/dq/tests/unit/dq/decorators/__init__.py b/providers/dataquality/tests/unit/dataquality/decorators/__init__.py similarity index 100% rename from providers/dq/tests/unit/dq/decorators/__init__.py rename to providers/dataquality/tests/unit/dataquality/decorators/__init__.py diff --git a/providers/dq/tests/unit/dq/decorators/test_dq_check.py b/providers/dataquality/tests/unit/dataquality/decorators/test_dq_check.py similarity index 92% rename from providers/dq/tests/unit/dq/decorators/test_dq_check.py rename to providers/dataquality/tests/unit/dataquality/decorators/test_dq_check.py index cbe88a1fb69f6..1c03e60cfbc01 100644 --- a/providers/dq/tests/unit/dq/decorators/test_dq_check.py +++ b/providers/dataquality/tests/unit/dataquality/decorators/test_dq_check.py @@ -21,9 +21,9 @@ import pytest from airflow.providers.common.sql.hooks.sql import DbApiHook -from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend -from airflow.providers.dq.decorators.dq_check import _DQCheckDecoratedOperator -from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dataquality.decorators.dq_check import _DQCheckDecoratedOperator +from airflow.providers.dataquality.rules import DQRule, RuleSet from airflow.sdk.execution_time.context import OutletEventAccessors NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) @@ -46,7 +46,9 @@ def make_context(): def results_backend(tmp_path): """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") - with mock.patch("airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend): + with mock.patch( + "airflow.providers.dataquality.operators.dq_check.get_backend_from_config", return_value=backend + ): yield backend diff --git a/providers/dq/tests/unit/dq/engines/__init__.py b/providers/dataquality/tests/unit/dataquality/engines/__init__.py similarity index 100% rename from providers/dq/tests/unit/dq/engines/__init__.py rename to providers/dataquality/tests/unit/dataquality/engines/__init__.py diff --git a/providers/dq/tests/unit/dq/engines/test_sql.py b/providers/dataquality/tests/unit/dataquality/engines/test_sql.py similarity index 97% rename from providers/dq/tests/unit/dq/engines/test_sql.py rename to providers/dataquality/tests/unit/dataquality/engines/test_sql.py index 9161275c3849a..2b5716673da07 100644 --- a/providers/dq/tests/unit/dq/engines/test_sql.py +++ b/providers/dataquality/tests/unit/dataquality/engines/test_sql.py @@ -21,8 +21,8 @@ import pytest from airflow.providers.common.sql.hooks.sql import DbApiHook -from airflow.providers.dq.engines.sql import SQLDQEngine -from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.providers.dataquality.engines.sql import SQLDQEngine +from airflow.providers.dataquality.rules import DQRule, RuleSet @pytest.fixture diff --git a/providers/dq/tests/unit/dq/operators/__init__.py b/providers/dataquality/tests/unit/dataquality/operators/__init__.py similarity index 100% rename from providers/dq/tests/unit/dq/operators/__init__.py rename to providers/dataquality/tests/unit/dataquality/operators/__init__.py diff --git a/providers/dq/tests/unit/dq/operators/test_dq_check.py b/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py similarity index 94% rename from providers/dq/tests/unit/dq/operators/test_dq_check.py rename to providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py index 9e74a8dc94642..6cb9642a2136a 100644 --- a/providers/dq/tests/unit/dq/operators/test_dq_check.py +++ b/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py @@ -21,11 +21,11 @@ import pytest from airflow.providers.common.sql.hooks.sql import DbApiHook -from airflow.providers.dq.assets import asset_quality -from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend -from airflow.providers.dq.exceptions import DQCheckFailedError -from airflow.providers.dq.operators.dq_check import DQCheckOperator -from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.providers.dataquality.assets import asset_quality +from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dataquality.exceptions import DQCheckFailedError +from airflow.providers.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.dataquality.rules import DQRule, RuleSet, Severity from airflow.sdk import Asset from airflow.sdk.execution_time.context import OutletEventAccessors @@ -62,7 +62,9 @@ def make_operator(records, **kwargs): def results_backend(tmp_path): """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") - with mock.patch("airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend): + with mock.patch( + "airflow.providers.dataquality.operators.dq_check.get_backend_from_config", return_value=backend + ): yield backend @@ -218,7 +220,7 @@ def test_backend_failure_does_not_fail_the_check(self, mock_get_db_hook): mock_get_db_hook.return_value = hook with mock.patch( - "airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend + "airflow.providers.dataquality.operators.dq_check.get_backend_from_config", return_value=backend ): summary = operator.execute(make_context()) diff --git a/providers/dq/tests/unit/dq/rules/__init__.py b/providers/dataquality/tests/unit/dataquality/rules/__init__.py similarity index 100% rename from providers/dq/tests/unit/dq/rules/__init__.py rename to providers/dataquality/tests/unit/dataquality/rules/__init__.py diff --git a/providers/dq/tests/unit/dq/rules/test_checks.py b/providers/dataquality/tests/unit/dataquality/rules/test_checks.py similarity index 95% rename from providers/dq/tests/unit/dq/rules/test_checks.py rename to providers/dataquality/tests/unit/dataquality/rules/test_checks.py index 8221a9f040da7..42a10fdd37331 100644 --- a/providers/dq/tests/unit/dq/rules/test_checks.py +++ b/providers/dataquality/tests/unit/dataquality/rules/test_checks.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from airflow.providers.dq.rules import CHECK_SPECS +from airflow.providers.dataquality.rules import CHECK_SPECS class TestCheckSpecs: diff --git a/providers/dq/tests/unit/dq/rules/test_rule.py b/providers/dataquality/tests/unit/dataquality/rules/test_rule.py similarity index 96% rename from providers/dq/tests/unit/dq/rules/test_rule.py rename to providers/dataquality/tests/unit/dataquality/rules/test_rule.py index 4c5cd46d38784..598317804e471 100644 --- a/providers/dq/tests/unit/dq/rules/test_rule.py +++ b/providers/dataquality/tests/unit/dataquality/rules/test_rule.py @@ -22,21 +22,21 @@ import pytest from pydantic import ValidationError -import airflow.providers.dq -from airflow.providers.dq.rules import Condition, DQRule, RuleSet, Severity, describe_rule +import airflow.providers.dataquality +from airflow.providers.dataquality.rules import Condition, DQRule, RuleSet, Severity, describe_rule def test_skill_reference_schema_matches_live_model(): """ - The dq-rule-authoring skill ships a generated RuleSet.model_json_schema() snapshot for + The dataquality-rule-authoring skill ships a generated RuleSet.model_json_schema() snapshot for agents to consult. Guard against it silently drifting out of sync with the real model -- regenerate with ``json.dumps(RuleSet.model_json_schema(), indent=2)`` (plus a trailing newline) if this fails after an intentional schema change. """ schema_path = ( - Path(airflow.providers.dq.__file__).parent + Path(airflow.providers.dataquality.__file__).parent / "skills" - / "dq-rule-authoring" + / "dataquality-rule-authoring" / "references" / "ruleset.schema.json" ) diff --git a/providers/dq/tests/unit/dq/test_assets.py b/providers/dataquality/tests/unit/dataquality/test_assets.py similarity index 97% rename from providers/dq/tests/unit/dq/test_assets.py rename to providers/dataquality/tests/unit/dataquality/test_assets.py index 9079b0ce567d9..4056f8ff288a3 100644 --- a/providers/dq/tests/unit/dq/test_assets.py +++ b/providers/dataquality/tests/unit/dataquality/test_assets.py @@ -18,7 +18,7 @@ import pytest -from airflow.providers.dq.assets import ( +from airflow.providers.dataquality.assets import ( DQ_EXTRA_KEY, DQ_RESULT_EXTRA_KEY, _quality_score_passes, @@ -27,8 +27,8 @@ get_asset_ruleset, require_quality, ) -from airflow.providers.dq.exceptions import DQRuleValidationError -from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.providers.dataquality.exceptions import DQRuleValidationError +from airflow.providers.dataquality.rules import DQRule, RuleSet from airflow.sdk import Asset from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult from airflow.sdk.execution_time.context import TriggeringAssetEventsAccessor diff --git a/providers/dq/tests/unit/dq/test_results.py b/providers/dataquality/tests/unit/dataquality/test_results.py similarity index 96% rename from providers/dq/tests/unit/dq/test_results.py rename to providers/dataquality/tests/unit/dataquality/test_results.py index 76b37ad1b1714..be43b18184fb4 100644 --- a/providers/dq/tests/unit/dq/test_results.py +++ b/providers/dataquality/tests/unit/dataquality/test_results.py @@ -18,7 +18,7 @@ import pytest -from airflow.providers.dq.results import DQRun, RuleResult, build_summary, compute_score +from airflow.providers.dataquality.results import DQRun, RuleResult, build_summary, compute_score def make_result(status: str, name: str = "r") -> RuleResult: diff --git a/pyproject.toml b/pyproject.toml index b61a987612779..58e62bd0a542a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,6 +209,9 @@ apache-airflow = "airflow.__main__:main" "datadog" = [ "apache-airflow-providers-datadog>=3.8.0" ] +"dataquality" = [ + "apache-airflow-providers-dataquality>=0.1.0" +] "dbt.cloud" = [ "apache-airflow-providers-dbt-cloud>=3.11.0" ] @@ -221,9 +224,6 @@ apache-airflow = "airflow.__main__:main" "docker" = [ "apache-airflow-providers-docker>=3.14.1" ] -"dq" = [ - "apache-airflow-providers-dq>=0.1.0" -] "edge3" = [ "apache-airflow-providers-edge3>=1.0.0" ] @@ -453,11 +453,11 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-common-sql>=1.18.0", "apache-airflow-providers-databricks>=6.11.0", "apache-airflow-providers-datadog>=3.8.0", + "apache-airflow-providers-dataquality>=0.1.0", "apache-airflow-providers-dbt-cloud>=3.11.0", "apache-airflow-providers-dingding>=3.7.0", "apache-airflow-providers-discord>=3.9.0", "apache-airflow-providers-docker>=3.14.1", - "apache-airflow-providers-dq>=0.1.0", "apache-airflow-providers-edge3>=1.0.0", "apache-airflow-providers-elasticsearch>=6.5.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-exasol>=4.6.1", @@ -1168,6 +1168,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/databricks/tests", "$MYPY_CONFIG_FILE_DIR/providers/datadog/src", "$MYPY_CONFIG_FILE_DIR/providers/datadog/tests", + "$MYPY_CONFIG_FILE_DIR/providers/dataquality/src", + "$MYPY_CONFIG_FILE_DIR/providers/dataquality/tests", "$MYPY_CONFIG_FILE_DIR/providers/dbt/cloud/src", "$MYPY_CONFIG_FILE_DIR/providers/dbt/cloud/tests", "$MYPY_CONFIG_FILE_DIR/providers/dingding/src", @@ -1176,8 +1178,6 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/discord/tests", "$MYPY_CONFIG_FILE_DIR/providers/docker/src", "$MYPY_CONFIG_FILE_DIR/providers/docker/tests", - "$MYPY_CONFIG_FILE_DIR/providers/dq/src", - "$MYPY_CONFIG_FILE_DIR/providers/dq/tests", "$MYPY_CONFIG_FILE_DIR/providers/edge3/src", "$MYPY_CONFIG_FILE_DIR/providers/edge3/tests", "$MYPY_CONFIG_FILE_DIR/providers/elasticsearch/src", @@ -1478,11 +1478,11 @@ apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false apache-airflow-providers-databricks = false apache-airflow-providers-datadog = false +apache-airflow-providers-dataquality = false apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false apache-airflow-providers-docker = false -apache-airflow-providers-dq = false apache-airflow-providers-edge3 = false apache-airflow-providers-elasticsearch = false apache-airflow-providers-exasol = false @@ -1631,11 +1631,11 @@ apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false apache-airflow-providers-databricks = false apache-airflow-providers-datadog = false +apache-airflow-providers-dataquality = false apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false apache-airflow-providers-docker = false -apache-airflow-providers-dq = false apache-airflow-providers-edge3 = false apache-airflow-providers-elasticsearch = false apache-airflow-providers-exasol = false @@ -1795,11 +1795,11 @@ apache-airflow-providers-common-messaging = { workspace = true } apache-airflow-providers-common-sql = { workspace = true } apache-airflow-providers-databricks = { workspace = true } apache-airflow-providers-datadog = { workspace = true } +apache-airflow-providers-dataquality = { workspace = true } apache-airflow-providers-dbt-cloud = { workspace = true } apache-airflow-providers-dingding = { workspace = true } apache-airflow-providers-discord = { workspace = true } apache-airflow-providers-docker = { workspace = true } -apache-airflow-providers-dq = { workspace = true } apache-airflow-providers-edge3 = { workspace = true } apache-airflow-providers-elasticsearch = { workspace = true } apache-airflow-providers-exasol = { workspace = true } @@ -1937,11 +1937,11 @@ members = [ "providers/common/sql", "providers/databricks", "providers/datadog", + "providers/dataquality", "providers/dbt/cloud", "providers/dingding", "providers/discord", "providers/docker", - "providers/dq", "providers/edge3", "providers/elasticsearch", "providers/exasol", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index 51afe0daa4df7..8578bd0488803 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -64,11 +64,11 @@ services: - ../../../empty:/opt/airflow/providers/common/sql/src - ../../../empty:/opt/airflow/providers/databricks/src - ../../../empty:/opt/airflow/providers/datadog/src + - ../../../empty:/opt/airflow/providers/dataquality/src - ../../../empty:/opt/airflow/providers/dbt/cloud/src - ../../../empty:/opt/airflow/providers/dingding/src - ../../../empty:/opt/airflow/providers/discord/src - ../../../empty:/opt/airflow/providers/docker/src - - ../../../empty:/opt/airflow/providers/dq/src - ../../../empty:/opt/airflow/providers/edge3/src - ../../../empty:/opt/airflow/providers/elasticsearch/src - ../../../empty:/opt/airflow/providers/exasol/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index e897b9b0ab948..1b48534a58c1f 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -77,11 +77,11 @@ services: - ../../../providers/common/sql/tests:/opt/airflow/providers/common/sql/tests - ../../../providers/databricks/tests:/opt/airflow/providers/databricks/tests - ../../../providers/datadog/tests:/opt/airflow/providers/datadog/tests + - ../../../providers/dataquality/tests:/opt/airflow/providers/dataquality/tests - ../../../providers/dbt/cloud/tests:/opt/airflow/providers/dbt/cloud/tests - ../../../providers/dingding/tests:/opt/airflow/providers/dingding/tests - ../../../providers/discord/tests:/opt/airflow/providers/discord/tests - ../../../providers/docker/tests:/opt/airflow/providers/docker/tests - - ../../../providers/dq/tests:/opt/airflow/providers/dq/tests - ../../../providers/edge3/tests:/opt/airflow/providers/edge3/tests - ../../../providers/elasticsearch/tests:/opt/airflow/providers/elasticsearch/tests - ../../../providers/exasol/tests:/opt/airflow/providers/exasol/tests diff --git a/scripts/ci/prek/generate_dq_ruleset_schema.py b/scripts/ci/prek/generate_dataquality_ruleset_schema.py similarity index 85% rename from scripts/ci/prek/generate_dq_ruleset_schema.py rename to scripts/ci/prek/generate_dataquality_ruleset_schema.py index 348c8d015b122..ad404c31016de 100755 --- a/scripts/ci/prek/generate_dq_ruleset_schema.py +++ b/scripts/ci/prek/generate_dataquality_ruleset_schema.py @@ -17,7 +17,7 @@ # under the License. """ Regenerate the Data Quality RuleSet JSON schema snapshot at -``providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json``. +``providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json``. """ from __future__ import annotations @@ -27,14 +27,14 @@ from common_prek_utils import AIRFLOW_ROOT_PATH, console -DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "dq" +DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "dataquality" SCHEMA_PATH = DQ_PROVIDER_PATH.joinpath( "src", "airflow", "providers", - "dq", + "dataquality", "skills", - "dq-rule-authoring", + "dataquality-rule-authoring", "references", "ruleset.schema.json", ) @@ -43,7 +43,7 @@ import json import sys -from airflow.providers.dq.rules import RuleSet +from airflow.providers.dataquality.rules import RuleSet sys.stdout.write(json.dumps(RuleSet.model_json_schema(), indent=2)) sys.stdout.write("\n") @@ -51,7 +51,7 @@ def dump_schema() -> str: - """Run the schema-dump snippet in the dq provider project and return its stdout.""" + """Run the schema-dump snippet in the dataquality provider project and return its stdout.""" result = subprocess.run( [ "uv", diff --git a/uv.lock b/uv.lock index b324a177b64ce..7753fd1df7ef6 100644 --- a/uv.lock +++ b/uv.lock @@ -53,6 +53,7 @@ apache-airflow-providers-telegram = false apache-airflow-providers-celery = false apache-airflow-providers-docker = false apache-airflow-providers-sendgrid = false +apache-airflow-providers-dataquality = false apache-airflow-providers-common-ai = false apache-airflow = false apache-airflow-shared-observability = false @@ -81,7 +82,6 @@ apache-airflow-providers-weaviate = false apache-airflow-providers-akeyless = false apache-airflow-providers-salesforce = false apache-airflow-providers-ssh = false -apache-airflow-providers-dq = false apache-airflow-providers-papermill = false apache-airflow-providers-google = false pydantic-ai-skills = "2026-07-16T00:00:00Z" @@ -212,11 +212,11 @@ members = [ "apache-airflow-providers-common-sql", "apache-airflow-providers-databricks", "apache-airflow-providers-datadog", + "apache-airflow-providers-dataquality", "apache-airflow-providers-dbt-cloud", "apache-airflow-providers-dingding", "apache-airflow-providers-discord", "apache-airflow-providers-docker", - "apache-airflow-providers-dq", "apache-airflow-providers-edge3", "apache-airflow-providers-elasticsearch", "apache-airflow-providers-exasol", @@ -1040,11 +1040,11 @@ all = [ { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-databricks" }, { name = "apache-airflow-providers-datadog" }, + { name = "apache-airflow-providers-dataquality" }, { name = "apache-airflow-providers-dbt-cloud" }, { name = "apache-airflow-providers-dingding" }, { name = "apache-airflow-providers-discord" }, { name = "apache-airflow-providers-docker" }, - { name = "apache-airflow-providers-dq" }, { name = "apache-airflow-providers-edge3" }, { name = "apache-airflow-providers-elasticsearch" }, { name = "apache-airflow-providers-exasol" }, @@ -1240,6 +1240,9 @@ databricks = [ datadog = [ { name = "apache-airflow-providers-datadog" }, ] +dataquality = [ + { name = "apache-airflow-providers-dataquality" }, +] dbt-cloud = [ { name = "apache-airflow-providers-dbt-cloud" }, ] @@ -1252,9 +1255,6 @@ discord = [ docker = [ { name = "apache-airflow-providers-docker" }, ] -dq = [ - { name = "apache-airflow-providers-dq" }, -] edge3 = [ { name = "apache-airflow-providers-edge3" }, ] @@ -1654,6 +1654,8 @@ requires-dist = [ { name = "apache-airflow-providers-databricks", marker = "extra == 'databricks'", editable = "providers/databricks" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'all'", editable = "providers/datadog" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'datadog'", editable = "providers/datadog" }, + { name = "apache-airflow-providers-dataquality", marker = "extra == 'all'", editable = "providers/dataquality" }, + { name = "apache-airflow-providers-dataquality", marker = "extra == 'dataquality'", editable = "providers/dataquality" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'all'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'dbt-cloud'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dingding", marker = "extra == 'all'", editable = "providers/dingding" }, @@ -1662,8 +1664,6 @@ requires-dist = [ { name = "apache-airflow-providers-discord", marker = "extra == 'discord'", editable = "providers/discord" }, { name = "apache-airflow-providers-docker", marker = "extra == 'all'", editable = "providers/docker" }, { name = "apache-airflow-providers-docker", marker = "extra == 'docker'", editable = "providers/docker" }, - { name = "apache-airflow-providers-dq", marker = "extra == 'all'", editable = "providers/dq" }, - { name = "apache-airflow-providers-dq", marker = "extra == 'dq'", editable = "providers/dq" }, { name = "apache-airflow-providers-edge3", marker = "extra == 'all'", editable = "providers/edge3" }, { name = "apache-airflow-providers-edge3", marker = "extra == 'edge3'", editable = "providers/edge3" }, { name = "apache-airflow-providers-elasticsearch", marker = "extra == 'all'", editable = "providers/elasticsearch" }, @@ -1802,7 +1802,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.28" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-dataquality", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "dq", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-dataquality", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dataquality", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -4952,6 +4952,49 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-dataquality" +version = "0.1.0" +source = { editable = "providers/dataquality" } +dependencies = [ + { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "pydantic", specifier = ">=2.11.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-dbt-cloud" version = "4.9.2" @@ -5124,49 +5167,6 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] -[[package]] -name = "apache-airflow-providers-dq" -version = "0.1.0" -source = { editable = "providers/dq" } -dependencies = [ - { name = "apache-airflow" }, - { name = "apache-airflow-providers-common-compat" }, - { name = "apache-airflow-providers-common-sql" }, - { name = "pydantic" }, - { name = "pyyaml" }, -] - -[package.dev-dependencies] -dev = [ - { name = "apache-airflow" }, - { name = "apache-airflow-devel-common" }, - { name = "apache-airflow-providers-common-compat" }, - { name = "apache-airflow-providers-common-sql" }, - { name = "apache-airflow-task-sdk" }, -] -docs = [ - { name = "apache-airflow-devel-common", extra = ["docs"] }, -] - -[package.metadata] -requires-dist = [ - { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, - { name = "pydantic", specifier = ">=2.11.0" }, - { name = "pyyaml", specifier = ">=6.0.2" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-devel-common", editable = "devel-common" }, - { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, - { name = "apache-airflow-task-sdk", editable = "task-sdk" }, -] -docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] - [[package]] name = "apache-airflow-providers-edge3" version = "4.1.0" From f77edaf6d6ebbddf77b44df1a7fcd7cbe0e2ac63 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Wed, 8 Jul 2026 10:51:57 +0100 Subject: [PATCH 08/19] Update references --- providers/dataquality/docs/assets.rst | 2 +- .../src/airflow/providers/dataquality/assets.py | 10 +++++----- .../tests/unit/dataquality/operators/test_dq_check.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/providers/dataquality/docs/assets.rst b/providers/dataquality/docs/assets.rst index ffbf797941fb0..2dddaabddd306 100644 --- a/providers/dataquality/docs/assets.rst +++ b/providers/dataquality/docs/assets.rst @@ -28,7 +28,7 @@ Attaching a ruleset to an asset ---------------------------------- :func:`~airflow.providers.dataquality.assets.asset_quality` stores ruleset, connection, and table -configuration inside ``Asset.extra`` under the ``airflow.dq`` key, so it is serialized with the +configuration inside ``Asset.extra`` under the ``airflow.dataquality`` key, so it is serialized with the Dag and needs no Airflow core changes: .. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_require_quality.py diff --git a/providers/dataquality/src/airflow/providers/dataquality/assets.py b/providers/dataquality/src/airflow/providers/dataquality/assets.py index 46020a5d23a6a..92af09155e1c3 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/assets.py +++ b/providers/dataquality/src/airflow/providers/dataquality/assets.py @@ -17,7 +17,7 @@ """ Asset-level data quality declarations. -Quality configuration lives inside ``Asset.extra`` under the ``airflow.dq`` key, so it is +Quality configuration lives inside ``Asset.extra`` under the ``airflow.dataquality`` key, so it is serialized with the Dag and needs no Airflow core changes. The rules travel with the asset definition instead of being scattered across the Dags that check it. """ @@ -39,8 +39,8 @@ log = logging.getLogger(__name__) -DQ_EXTRA_KEY = "airflow.dq" -DQ_RESULT_EXTRA_KEY = "airflow.dq.result" +DQ_EXTRA_KEY = "airflow.dataquality" +DQ_RESULT_EXTRA_KEY = "airflow.dataquality.result" def asset_quality( @@ -83,7 +83,7 @@ def asset_quality( def get_asset_quality_config(asset: Asset) -> dict[str, Any] | None: - """Return the raw ``airflow.dq`` config attached to an asset, if any.""" + """Return the raw ``airflow.dataquality`` config attached to an asset, if any.""" config = asset.extra.get(DQ_EXTRA_KEY) if not isinstance(config, dict): return None @@ -145,7 +145,7 @@ def require_quality( Gate a Dag run on the data quality score attached to one of its triggering asset events. Reads the summary a :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` - attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dq.result"]`` + attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dataquality.result"]`` (see :func:`asset_quality`), and short-circuits the run — skipping every downstream task — when that summary is missing or its score is below ``min_score``. Call it inside a Dag scheduled by ``asset``:: diff --git a/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py b/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py index 6cb9642a2136a..005082bce0f2e 100644 --- a/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py +++ b/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py @@ -246,4 +246,4 @@ def test_summary_attached_to_outlet_events(self, mock_get_db_hook): outlet_events = context["outlet_events"] outlet_events.__getitem__.assert_called_with(asset) extra = outlet_events.__getitem__.return_value.extra - extra.__setitem__.assert_called_once_with("airflow.dq.result", summary) + extra.__setitem__.assert_called_once_with("airflow.dataquality.result", summary) From 734307b58bce2881ca21961f8376a3781ea4dd23 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Thu, 9 Jul 2026 21:22:45 +0100 Subject: [PATCH 09/19] rename provider to common-dataquality --- .../ISSUE_TEMPLATE/1-airflow_bug_report.yml | 1 - .github/boring-cyborg.yml | 4 +- airflow-core/docs/extra-packages-ref.rst | 2 - .../unit/always/test_project_structure.py | 2 +- dev/breeze/doc/images/output_build-docs.svg | 16 +- dev/breeze/doc/images/output_build-docs.txt | 2 +- ...release-management_add-back-references.svg | 16 +- ...release-management_add-back-references.txt | 2 +- ...e-management_classify-provider-changes.svg | 16 +- ...e-management_classify-provider-changes.txt | 2 +- ...ement_generate-issue-content-providers.svg | 16 +- ...ement_generate-issue-content-providers.txt | 2 +- ...management_generate-providers-metadata.svg | 18 +- ...management_generate-providers-metadata.txt | 2 +- ...agement_prepare-provider-distributions.svg | 16 +- ...agement_prepare-provider-distributions.txt | 2 +- ...agement_prepare-provider-documentation.svg | 16 +- ...agement_prepare-provider-documentation.txt | 2 +- ...output_release-management_publish-docs.svg | 16 +- ...output_release-management_publish-docs.txt | 2 +- ...t_sbom_generate-providers-requirements.svg | 18 +- ...t_sbom_generate-providers-requirements.txt | 2 +- .../output_workflow-run_publish-docs.svg | 16 +- .../output_workflow-run_publish-docs.txt | 2 +- dev/breeze/tests/test_selective_checks.py | 2 +- .../dataquality/.pre-commit-config.yaml | 8 +- providers/common/dataquality/README.rst | 20 +- .../{ => common}/dataquality/docs/agents.rst | 8 +- .../{ => common}/dataquality/docs/assets.rst | 10 +- .../dataquality/docs/configurations-ref.rst | 4 +- .../dataquality/docs/decorators.rst | 8 +- providers/common/dataquality/docs/index.rst | 46 ++-- .../dataquality/docs/operators.rst | 20 +- .../{ => common}/dataquality/docs/rules.rst | 20 +- providers/common/dataquality/provider.yaml | 47 +++- providers/common/dataquality/pyproject.toml | 9 +- .../providers/common}/dataquality/assets.py | 8 +- .../common}/dataquality/backends/__init__.py | 8 +- .../dataquality/backends/object_storage.py | 2 +- .../dataquality/decorators/__init__.py | 0 .../dataquality/decorators/dq_check.py | 8 +- .../common}/dataquality/engines/__init__.py | 2 +- .../common}/dataquality/engines/sql.py | 6 +- .../dataquality/example_dags/__init__.py | 0 .../example_dq_check_custom_sql.py | 4 +- .../example_dq_check_decorator_dynamic.py | 2 +- .../example_dq_llm_generated_ruleset.py | 2 +- .../example_dq_require_quality.py | 12 +- .../example_dq_ruleset_from_yaml.py | 8 +- .../example_dags/orders_ruleset.yaml | 0 .../common}/dataquality/exceptions.py | 0 .../common/dataquality/get_provider_info.py | 45 +++- .../common}/dataquality/operators/__init__.py | 2 +- .../common}/dataquality/operators/dq_check.py | 32 +-- .../providers/common}/dataquality/results.py | 0 .../common}/dataquality/rules/__init__.py | 4 +- .../common}/dataquality/rules/checks.py | 0 .../common}/dataquality/rules/rule.py | 4 +- .../dataquality-rule-authoring/SKILL.md | 0 .../references/ruleset.schema.json | 0 .../common}/dataquality/version_compat.py | 0 .../common}/dataquality/example_dq_check.py | 10 +- .../common/dataquality/backends}/__init__.py | 0 .../backends/test_object_storage.py | 12 +- .../dataquality/decorators}/__init__.py | 0 .../dataquality/decorators/test_dq_check.py | 9 +- .../common/dataquality/engines}/__init__.py | 0 .../common}/dataquality/engines/test_sql.py | 4 +- .../common/dataquality/operators}/__init__.py | 0 .../dataquality/operators/test_dq_check.py | 16 +- .../common/dataquality/rules}/__init__.py | 0 .../common}/dataquality/rules/test_checks.py | 2 +- .../common}/dataquality/rules/test_rule.py | 6 +- .../unit/common}/dataquality/test_assets.py | 6 +- .../unit/common}/dataquality/test_results.py | 2 +- providers/dataquality/.gitignore | 3 - providers/dataquality/LICENSE | 201 ------------------ providers/dataquality/NOTICE | 5 - providers/dataquality/README.rst | 52 ----- providers/dataquality/docs/changelog.rst | 33 --- providers/dataquality/docs/commits.rst | 35 --- providers/dataquality/docs/conf.py | 27 --- providers/dataquality/docs/index.rst | 175 --------------- .../installing-providers-from-sources.rst | 18 -- providers/dataquality/docs/security.rst | 18 -- providers/dataquality/provider.yaml | 77 ------- providers/dataquality/pyproject.toml | 128 ----------- providers/dataquality/src/airflow/__init__.py | 17 -- .../src/airflow/providers/__init__.py | 17 -- .../airflow/providers/dataquality/__init__.py | 39 ---- .../dataquality/get_provider_info.py | 70 ------ providers/dataquality/tests/conftest.py | 19 -- .../dataquality/tests/system/__init__.py | 17 -- providers/dataquality/tests/unit/__init__.py | 17 -- .../unit/dataquality/operators/__init__.py | 16 -- .../tests/unit/dataquality/rules/__init__.py | 16 -- pyproject.toml | 10 - scripts/ci/docker-compose/remove-sources.yml | 1 - scripts/ci/docker-compose/tests-sources.yml | 1 - ...rate_common_dataquality_ruleset_schema.py} | 7 +- uv.lock | 75 +------ 101 files changed, 363 insertions(+), 1342 deletions(-) rename providers/{ => common}/dataquality/.pre-commit-config.yaml (77%) rename providers/{ => common}/dataquality/docs/agents.rst (82%) rename providers/{ => common}/dataquality/docs/assets.rst (82%) rename providers/{ => common}/dataquality/docs/configurations-ref.rst (81%) rename providers/{ => common}/dataquality/docs/decorators.rst (83%) rename providers/{ => common}/dataquality/docs/operators.rst (81%) rename providers/{ => common}/dataquality/docs/rules.rst (89%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/assets.py (94%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/backends/__init__.py (75%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/backends/object_storage.py (99%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/decorators/__init__.py (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/decorators/dq_check.py (91%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/engines/__init__.py (90%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/engines/sql.py (96%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/example_dags/__init__.py (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/example_dags/example_dq_check_custom_sql.py (95%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/example_dags/example_dq_check_decorator_dynamic.py (97%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/example_dags/example_dq_llm_generated_ruleset.py (98%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/example_dags/example_dq_require_quality.py (87%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/example_dags/example_dq_ruleset_from_yaml.py (89%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/example_dags/orders_ruleset.yaml (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/exceptions.py (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/operators/__init__.py (91%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/operators/dq_check.py (90%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/results.py (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/rules/__init__.py (87%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/rules/checks.py (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/rules/rule.py (98%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/skills/dataquality-rule-authoring/SKILL.md (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json (100%) rename providers/{dataquality/src/airflow/providers => common/dataquality/src/airflow/providers/common}/dataquality/version_compat.py (100%) rename providers/{dataquality/tests/system => common/dataquality/tests/system/common}/dataquality/example_dq_check.py (96%) rename providers/{dataquality/tests/system/dataquality => common/dataquality/tests/unit/common/dataquality/backends}/__init__.py (100%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/backends/test_object_storage.py (96%) rename providers/{dataquality/tests/unit/dataquality => common/dataquality/tests/unit/common/dataquality/decorators}/__init__.py (100%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/decorators/test_dq_check.py (92%) rename providers/{dataquality/tests/unit/dataquality/backends => common/dataquality/tests/unit/common/dataquality/engines}/__init__.py (100%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/engines/test_sql.py (97%) rename providers/{dataquality/tests/unit/dataquality/decorators => common/dataquality/tests/unit/common/dataquality/operators}/__init__.py (100%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/operators/test_dq_check.py (93%) rename providers/{dataquality/tests/unit/dataquality/engines => common/dataquality/tests/unit/common/dataquality/rules}/__init__.py (100%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/rules/test_checks.py (94%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/rules/test_rule.py (97%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/test_assets.py (96%) rename providers/{dataquality/tests/unit => common/dataquality/tests/unit/common}/dataquality/test_results.py (96%) delete mode 100644 providers/dataquality/.gitignore delete mode 100644 providers/dataquality/LICENSE delete mode 100644 providers/dataquality/NOTICE delete mode 100644 providers/dataquality/README.rst delete mode 100644 providers/dataquality/docs/changelog.rst delete mode 100644 providers/dataquality/docs/commits.rst delete mode 100644 providers/dataquality/docs/conf.py delete mode 100644 providers/dataquality/docs/index.rst delete mode 100644 providers/dataquality/docs/installing-providers-from-sources.rst delete mode 100644 providers/dataquality/docs/security.rst delete mode 100644 providers/dataquality/provider.yaml delete mode 100644 providers/dataquality/pyproject.toml delete mode 100644 providers/dataquality/src/airflow/__init__.py delete mode 100644 providers/dataquality/src/airflow/providers/__init__.py delete mode 100644 providers/dataquality/src/airflow/providers/dataquality/__init__.py delete mode 100644 providers/dataquality/src/airflow/providers/dataquality/get_provider_info.py delete mode 100644 providers/dataquality/tests/conftest.py delete mode 100644 providers/dataquality/tests/system/__init__.py delete mode 100644 providers/dataquality/tests/unit/__init__.py delete mode 100644 providers/dataquality/tests/unit/dataquality/operators/__init__.py delete mode 100644 providers/dataquality/tests/unit/dataquality/rules/__init__.py rename scripts/ci/prek/{generate_dataquality_ruleset_schema.py => generate_common_dataquality_ruleset_schema.py} (89%) diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index babfd2b0df5d3..50b2399564e0d 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -149,7 +149,6 @@ body: - common-sql - databricks - datadog - - dataquality - dbt-cloud - dingding - discord diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 22492c7ebf0b4..5a8b8179fba9e 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -144,8 +144,8 @@ labelPRBasedOnFilePath: provider:docker: - providers/docker/** - provider:dataquality: - - providers/dataquality/** + provider:common-dataquality: + - providers/common/dataquality/** provider:edge: - providers/edge3/** diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index 4e9b6c95b6400..b6617690de233 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -261,8 +261,6 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | datadog | ``pip install 'apache-airflow[datadog]'`` | Datadog hooks and sensors | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ -| dataquality | ``pip install 'apache-airflow[dataquality]'`` | Data Quality hooks and operators | -+---------------------+-----------------------------------------------------+-----------------------------------------------------+ | dbt-cloud | ``pip install 'apache-airflow[dbt-cloud]'`` | dbt Cloud hooks and operators | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | dingding | ``pip install 'apache-airflow[dingding]'`` | Dingding hooks and sensors | diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py index 0782f836eb09d..c5f51a8884111 100644 --- a/airflow-core/tests/unit/always/test_project_structure.py +++ b/airflow-core/tests/unit/always/test_project_structure.py @@ -107,7 +107,7 @@ def test_providers_modules_should_have_tests(self): "providers/common/compat/tests/unit/common/compat/standard/test_utils.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_base_provider.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_sqs.py", - "providers/dataquality/tests/unit/dataquality/test_exceptions.py", + "providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py", "providers/edge3/tests/unit/edge3/cli/test_example_extended_sysinfo.py", "providers/edge3/tests/unit/edge3/models/test_edge_job.py", "providers/edge3/tests/unit/edge3/models/test_edge_logs.py", diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 8214bae02bb6b..99bc5dbc0de1e 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -244,14 +244,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality -| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git -| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 592af20a3509a..75dd9355bcff2 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -650f0de60cf3fdf7767dfd40b91a2675 +e22969fb5e92a3efafb744881fea5ef8 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 9a44dfb4c583c..9df4462030484 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -153,14 +153,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality -| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git -| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index b92168b40a253..152f412a472d3 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -ff9e85fd0e4ce1969fc862af15407a99 +963d8b3e84cb64aa0d69026638b9fb1b diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index f02970470843f..7e2dae4729d23 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -165,14 +165,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc -| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 96dac2cdc561b..c400d2c1da050 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -9d3d9f020483dfb9d90b55c2a1d717ee +b4c6b4c23a30b7f750a9a8316abd2e91 diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index c4b637d2ab322..78c943d5732d6 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -154,14 +154,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc -| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 8c7b6cbd28587..7c5cc1b0fb397 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -6f895598495ed23ae78b9a1ca070d041 +ef90cd93a64627b874f63cca845bddbf diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 130bb12f325a6..4199019d5c56f 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -173,15 +173,15 @@ │apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | â”‚ │apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | â”‚ │apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ -│| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | â”‚ -│datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol│ -│| fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | â”‚ -│informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | â”‚ -│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | â”‚ -│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | â”‚ -│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake â”‚ -│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ -│yandex | ydb | zendesk)│ +│| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | â”‚ +│common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | â”‚ +│elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http â”‚ +│| imap | influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql â”‚ +│| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | â”‚ +│openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | â”‚ +│postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | â”‚ +│singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ +│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ │--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    â”‚ │running metadata generation to only this version of the provider. (TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index b5a1424d3aa36..bf322a8137ed0 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -4df98abc0b2a412910bb4ef2560599fd +0e161f1be5f2a5b8c6094dcf7859f1c8 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 5d1d0e44a2248..2a119b59f74a7 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -192,14 +192,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc -| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index 091e95e9aae8a..9cc84f8ed57aa 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -7c2e498681b64e4a1357bc4003862c8d +40982121f81ca247791eee99e22d7214 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index a783d6e452406..4785a550ef45e 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -213,14 +213,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dataquality | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch |    -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc -| jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |  -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 7d081d484eae2..cab1aa0379851 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -2f2b0ac5ad9d4337ac7ac645a182b2d2 +0d1c20ffd24edc17351c5f132a1cc968 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index 043afef3dfeaf..59c03ad4d408a 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -192,14 +192,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality -| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git -| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index d2c80bf83297d..585d41d6275cb 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -794459df3168cc9d0ea3b4fdb12ada1f +217c089e723c5584da63135bbf011504 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index 0c51c26d64dd4..8e1ee534c9b46 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -187,15 +187,15 @@ │apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | â”‚ │apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | â”‚ │atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | â”‚ -│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality |│ -│dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp â”‚ -│| git | github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | â”‚ -│jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm│ -│| mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | â”‚ -│oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | â”‚ -│salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite│ -│| ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |│ -│ydb | zendesk)│ +│common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks | â”‚ +│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | â”‚ +│facebook | ftp | git | github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | â”‚ +│informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | â”‚ +│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | â”‚ +│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | â”‚ +│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake â”‚ +│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ +│yandex | ydb | zendesk)│ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported     â”‚ │value to account for the most recent version of the provider (TEXT)│ │--force           Force update providers requirements even if they already exist.│ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index c4db876f000f5..ca3930b91a1a2 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -74188d0a9923aa0a11de67eca11cca65 +7cbc3f1d6c0c4f1745702d867f9e7fdc diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 15f8fa38e1233..13e7e9bf14620 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -207,14 +207,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dataquality -| dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git -| github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc |  -jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    -odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone -| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 9cbe2fb415404..a86438aee4e27 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -30ce85f34479a082fd77fb47b9557688 +cb87f41a19fbc8895cd38aa6351139ae diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 7510add14cd38..691ec69550b82 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2884,7 +2884,7 @@ def test_upgrade_to_newer_dependencies( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " - "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks dataquality elasticsearch " + "apache.impala apache.pinot clickhousedb common.ai common.compat common.dataquality common.sql databricks elasticsearch " "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", }, diff --git a/providers/dataquality/.pre-commit-config.yaml b/providers/common/dataquality/.pre-commit-config.yaml similarity index 77% rename from providers/dataquality/.pre-commit-config.yaml rename to providers/common/dataquality/.pre-commit-config.yaml index 5a509f51908f1..173d5a5527520 100644 --- a/providers/dataquality/.pre-commit-config.yaml +++ b/providers/common/dataquality/.pre-commit-config.yaml @@ -24,13 +24,13 @@ default_language_version: repos: - repo: local hooks: - - id: generate-dataquality-ruleset-schema + - id: generate-common-dataquality-ruleset-schema name: Generate Data Quality RuleSet schema language: python - entry: ../../scripts/ci/prek/generate_dataquality_ruleset_schema.py + entry: ../../../scripts/ci/prek/generate_common_dataquality_ruleset_schema.py pass_filenames: false always_run: true files: > (?x) - ^src/airflow/providers/dataquality/rules/.*\.py$| - ^src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset\.schema\.json$ + ^src/airflow/providers/common/dataquality/rules/.*\.py$| + ^src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset\.schema\.json$ diff --git a/providers/common/dataquality/README.rst b/providers/common/dataquality/README.rst index baddbee922875..b9e552d12d263 100644 --- a/providers/common/dataquality/README.rst +++ b/providers/common/dataquality/README.rst @@ -19,7 +19,11 @@ Package ``apache-airflow-providers-common-dataquality`` Release: ``0.1.0`` -Common Data Quality Provider +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. Provider package ---------------- @@ -37,8 +41,12 @@ see ``Requirements`` below. Requirements ------------ -================== ================== -PIP package Version required -================== ================== -``apache-airflow`` ``>=3.0.0`` -================== ================== +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== diff --git a/providers/dataquality/docs/agents.rst b/providers/common/dataquality/docs/agents.rst similarity index 82% rename from providers/dataquality/docs/agents.rst rename to providers/common/dataquality/docs/agents.rst index 355c07546f632..8a0d160a4e2f2 100644 --- a/providers/dataquality/docs/agents.rst +++ b/providers/common/dataquality/docs/agents.rst @@ -20,7 +20,7 @@ Generating rules with an LLM ============================== -Writing a :class:`~airflow.providers.dataquality.rules.RuleSet` by hand for every table doesn't scale. +Writing a :class:`~airflow.providers.common.dataquality.rules.RuleSet` by hand for every table doesn't scale. An LLM can propose one from a table's column definitions instead, given the check catalog as context. @@ -28,7 +28,7 @@ The ``dataquality-rule-authoring`` skill ---------------------------------------- This provider ships an `Agent Skill `__ at -``airflow/providers/dataquality/skills/dataquality-rule-authoring/``: a ``SKILL.md`` documenting the +``airflow/providers/common/dataquality/skills/dataquality-rule-authoring/``: a ``SKILL.md`` documenting the ``RuleSet``/``DQRule`` fields and check catalog, plus a generated JSON Schema (``references/ruleset.schema.json``) for validation. @@ -36,7 +36,7 @@ Point ``common.ai``'s :doc:`AgentSkillsToolset + Rules, rule sets, and checks + Operators + Decorators + Assets and quality gating + Generating rules with an LLM .. toctree:: :hidden: :maxdepth: 1 - :caption: System tests + :caption: References - System Tests <_api/tests/system/common/dataquality/index> + Configuration + Python API <_api/airflow/providers/common/dataquality/index> .. toctree:: :hidden: @@ -50,21 +55,12 @@ PyPI Repository Installing from sources -.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Commits - - Detailed list of commits - - -apache-airflow-providers-common-dataquality package ------------------------------------------------------- +``Data Quality Provider`` -Common Data Quality Provider +Declarative data quality rules with durable, per-rule execution history. +Checks run through ``common.sql`` DB-API hooks; results are persisted to a +configurable results store (object storage or local files) so task, run, +and rule-level quality can be inspected over time. Release: 0.1.0 @@ -87,11 +83,15 @@ Requirements The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. -================== ================== -PIP package Version required -================== ================== -``apache-airflow`` ``>=3.0.0`` -================== ================== +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== Downloading official packages ----------------------------- diff --git a/providers/dataquality/docs/operators.rst b/providers/common/dataquality/docs/operators.rst similarity index 81% rename from providers/dataquality/docs/operators.rst rename to providers/common/dataquality/docs/operators.rst index 72a357ee4fda6..ab5eec1669b74 100644 --- a/providers/dataquality/docs/operators.rst +++ b/providers/common/dataquality/docs/operators.rst @@ -20,7 +20,7 @@ ``DQCheckOperator`` ===================== -Use :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` to run a +Use :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` to run a :doc:`ruleset ` against a table and persist per-rule results to the configured results store. Every rule is evaluated and recorded regardless of the task outcome -- a rule failing doesn't stop the others from running, and the full set of results is always written before the @@ -31,7 +31,7 @@ Basic usage Pass a connection, table, and ruleset directly: -.. exampleinclude:: /../tests/system/dataquality/example_dq_check.py +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py :language: python :dedent: 4 :start-after: [START howto_operator_dq_check] @@ -40,12 +40,12 @@ Pass a connection, table, and ruleset directly: Parameters ----------- -- ``ruleset`` -- a :class:`~airflow.providers.dataquality.rules.RuleSet`, its dict form, or a path to a +- ``ruleset`` -- a :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path to a YAML ruleset file (see :doc:`rules`). Optional when ``asset`` carries one. - ``table`` -- the table to check. Optional when ``asset`` carries one (falling back to the asset's name). - ``asset`` -- an :class:`~airflow.sdk.Asset` decorated with - :func:`~airflow.providers.dataquality.assets.asset_quality`. Supplies defaults for ``ruleset``, + :func:`~airflow.providers.common.dataquality.assets.asset_quality`. Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's outlets so its asset events carry the check summary -- see :doc:`assets`. - ``partition_clause`` -- predicate ANDed into every built-in check's ``WHERE`` clause, e.g. @@ -69,7 +69,7 @@ can't even run. Persisting results -------------------- -Results are persisted to the backend configured under ``[dataquality] results_path`` (see +Results are persisted to the backend configured under ``[common.dataquality] results_path`` (see :doc:`configurations-ref`). When that's unset, checks still run and the task still passes or fails normally -- only persisted data quality history is unavailable. There is no per-operator override: every check in a deployment shares one results store, so history stays @@ -79,17 +79,17 @@ Checking an asset -------------------- Attach a ruleset to an :class:`~airflow.sdk.Asset` with -:func:`~airflow.providers.dataquality.assets.asset_quality`, then pass the asset instead of ``table``/ +:func:`~airflow.providers.common.dataquality.assets.asset_quality`, then pass the asset instead of ``table``/ ``ruleset``/``conn_id``: -.. exampleinclude:: /../tests/system/dataquality/example_dq_check.py +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py :language: python :start-after: [START howto_operator_dq_check_asset] :end-before: [END howto_operator_dq_check_asset] The operator adds the asset to its own outlets automatically, and attaches the check's summary (including its quality ``score``) to the asset event -- which is what makes -:func:`~airflow.providers.dataquality.assets.require_quality` (see :doc:`assets`) able to gate a +:func:`~airflow.providers.common.dataquality.assets.require_quality` (see :doc:`assets`) able to gate a downstream consumer Dag on it. ``custom_sql`` checks @@ -98,7 +98,7 @@ downstream consumer Dag on it. Built-in checks are all single-column. For cross-column comparisons, joins, or anything the catalog doesn't cover, use ``custom_sql``: -.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py :language: python :start-after: [START howto_operator_dq_check_custom_sql] :end-before: [END howto_operator_dq_check_custom_sql] @@ -108,7 +108,7 @@ See :doc:`rules` for the full built-in check catalog and the ``custom_sql`` gram Loading a ruleset from YAML ------------------------------ -.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py :language: python :start-after: [START howto_operator_dq_check_ruleset_from_yaml] :end-before: [END howto_operator_dq_check_ruleset_from_yaml] diff --git a/providers/dataquality/docs/rules.rst b/providers/common/dataquality/docs/rules.rst similarity index 89% rename from providers/dataquality/docs/rules.rst rename to providers/common/dataquality/docs/rules.rst index ac1e76d71ac32..9ee6b8fd0a254 100644 --- a/providers/dataquality/docs/rules.rst +++ b/providers/common/dataquality/docs/rules.rst @@ -20,15 +20,15 @@ Rules, rule sets, and checks ============================ -Rules are data, not code. A :class:`~airflow.providers.dataquality.rules.RuleSet` is a named tuple of -:class:`~airflow.providers.dataquality.rules.DQRule` -- plain, serializable objects with no behavior of -their own. They describe *what* to check; :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` +Rules are data, not code. A :class:`~airflow.providers.common.dataquality.rules.RuleSet` is a named tuple of +:class:`~airflow.providers.common.dataquality.rules.DQRule` -- plain, serializable objects with no behavior of +their own. They describe *what* to check; :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` (see :doc:`operators`) is what actually runs them. Rules are usually written by hand, but they can also be proposed by an LLM -- see :doc:`agents`. .. code-block:: python - from airflow.providers.dataquality.rules import DQRule, RuleSet + from airflow.providers.common.dataquality.rules import DQRule, RuleSet orders_ruleset = RuleSet( name="orders_quality", @@ -130,7 +130,7 @@ column-level checks: Conditions ----------- -A :class:`~airflow.providers.dataquality.rules.Condition` is the pass/fail rule applied to the observed +A :class:`~airflow.providers.common.dataquality.rules.Condition` is the pass/fail rule applied to the observed value, using the same grammar as the ``common.sql`` check operators: - ``equal_to`` -- exact match. Cannot be combined with any other comparison. @@ -149,7 +149,7 @@ Built-in checks are all single-column. The moment a rule needs to compare two co another table, or use a function the catalog doesn't cover, use ``custom_sql`` -- any SQL statement that resolves to a single scalar, evaluated exactly like a built-in check: -.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py :language: python :start-after: [START howto_operator_dq_check_custom_sql] :end-before: [END howto_operator_dq_check_custom_sql] @@ -186,14 +186,14 @@ Loading rules from YAML ------------------------- Anywhere a ``RuleSet`` is accepted -- ``DQCheckOperator(ruleset=...)``, -``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.dataquality.assets.asset_quality` -- a path -string is accepted too, and resolved via :meth:`~airflow.providers.dataquality.rules.RuleSet.from_file` +``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.common.dataquality.assets.asset_quality` -- a path +string is accepted too, and resolved via :meth:`~airflow.providers.common.dataquality.rules.RuleSet.from_file` at Dag-parse time. This keeps rules editable by people who don't write Python. -.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/orders_ruleset.yaml +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml :language: yaml -.. exampleinclude:: /../src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py :language: python :start-after: [START howto_operator_dq_check_ruleset_from_yaml] :end-before: [END howto_operator_dq_check_ruleset_from_yaml] diff --git a/providers/common/dataquality/provider.yaml b/providers/common/dataquality/provider.yaml index ae44126145dd5..80969a57621e7 100644 --- a/providers/common/dataquality/provider.yaml +++ b/providers/common/dataquality/provider.yaml @@ -17,9 +17,14 @@ --- package-name: apache-airflow-providers-common-dataquality -name: Common Data Quality +name: Data Quality description: | - Common Data Quality Provider + ``Data Quality Provider`` + + Declarative data quality rules with durable, per-rule execution history. + Checks run through ``common.sql`` DB-API hooks; results are persisted to a + configurable results store (object storage or local files) so task, run, + and rule-level quality can be inspected over time. state: ready lifecycle: incubation @@ -32,3 +37,41 @@ build-system: flit_core # to be done in the same PR versions: - 0.1.0 + +integrations: + - integration-name: Data Quality + external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/ + how-to-guide: + - /docs/apache-airflow-providers-common-dataquality/operators.rst + tags: [software] + +operators: + - integration-name: Data Quality + python-modules: + - airflow.providers.common.dataquality.operators.dq_check + +task-decorators: + - class-name: airflow.providers.common.dataquality.decorators.dq_check.dq_check_task + name: dq_check + +config: + common.dataquality: + description: | + Configuration for the Data Quality provider results store. + options: + results_path: + description: | + Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality + results are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``. + When unset, checks still run but no history is persisted. + version_added: 0.1.0 + type: string + example: s3://data-platform/airflow-dq + default: ~ + results_conn_id: + description: | + Optional Airflow connection used to access ``results_path``. + version_added: 0.1.0 + type: string + example: aws_default + default: ~ diff --git a/providers/common/dataquality/pyproject.toml b/providers/common/dataquality/pyproject.toml index fd65bbbbd02ef..b8c4e6f6b5dd9 100644 --- a/providers/common/dataquality/pyproject.toml +++ b/providers/common/dataquality/pyproject.toml @@ -60,13 +60,8 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=3.0.0", -] - -[dependency-groups] -dev = [ - "apache-airflow", - "apache-airflow-task-sdk", - "apache-airflow-devel-common", + "apache-airflow-providers-common-compat", + "apache-airflow-providers-common-sql", # Additional devel dependencies (do not remove this line and add extra development dependencies) ] diff --git a/providers/dataquality/src/airflow/providers/dataquality/assets.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py similarity index 94% rename from providers/dataquality/src/airflow/providers/dataquality/assets.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py index 92af09155e1c3..4655e9478ab89 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/assets.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py @@ -27,8 +27,8 @@ import logging from typing import TYPE_CHECKING, Any -from airflow.providers.dataquality.exceptions import DQRuleValidationError -from airflow.providers.dataquality.rules import RuleSet +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules import RuleSet from airflow.sdk import task if TYPE_CHECKING: @@ -54,7 +54,7 @@ def asset_quality( Attach data quality configuration to an asset, returning the same asset. :param asset: The asset the rules describe. - :param ruleset: A :class:`~airflow.providers.dataquality.rules.RuleSet`, its dict form, or a path + :param ruleset: A :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path to a YAML ruleset file (resolved eagerly, at Dag-parse time). :param conn_id: Default connection a check operator should use for this asset. :param table: Default table to check; falls back to the asset name when unset. @@ -144,7 +144,7 @@ def require_quality( """ Gate a Dag run on the data quality score attached to one of its triggering asset events. - Reads the summary a :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` + Reads the summary a :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dataquality.result"]`` (see :func:`asset_quality`), and short-circuits the run — skipping every downstream task — when that summary is missing or its score is below ``min_score``. Call it inside a Dag diff --git a/providers/dataquality/src/airflow/providers/dataquality/backends/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py similarity index 75% rename from providers/dataquality/src/airflow/providers/dataquality/backends/__init__.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py index 58e7df4024d93..a41919c9f627a 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/backends/__init__.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py @@ -16,17 +16,17 @@ # under the License. from __future__ import annotations -from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend __all__ = ["ObjectStorageResultsBackend", "get_backend_from_config"] def get_backend_from_config() -> ObjectStorageResultsBackend | None: - """Build the results backend configured under ``[dataquality]``, or ``None`` when not configured.""" + """Build the results backend configured under ``[common.dataquality]``, or ``None`` when not configured.""" from airflow.providers.common.compat.sdk import conf - results_path = conf.get("dataquality", "results_path", fallback=None) + results_path = conf.get("common.dataquality", "results_path", fallback=None) if not results_path: return None - conn_id = conf.get("dataquality", "results_conn_id", fallback=None) + conn_id = conf.get("common.dataquality", "results_conn_id", fallback=None) return ObjectStorageResultsBackend(results_path=results_path, conn_id=conn_id or None) diff --git a/providers/dataquality/src/airflow/providers/dataquality/backends/object_storage.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py similarity index 99% rename from providers/dataquality/src/airflow/providers/dataquality/backends/object_storage.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py index 94d0236f975db..16b963548b094 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/backends/object_storage.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py @@ -39,7 +39,7 @@ from datetime import datetime, timezone from typing import Any -from airflow.providers.dataquality.results import DQRun, RuleResult, build_summary +from airflow.providers.common.dataquality.results import DQRun, RuleResult, build_summary from airflow.sdk import ObjectStoragePath log = logging.getLogger(__name__) diff --git a/providers/dataquality/src/airflow/providers/dataquality/decorators/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/decorators/__init__.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py diff --git a/providers/dataquality/src/airflow/providers/dataquality/decorators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py similarity index 91% rename from providers/dataquality/src/airflow/providers/dataquality/decorators/dq_check.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py index 2da6bf423717a..6bd05a2e08e8f 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/decorators/dq_check.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""TaskFlow decorator for :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator`.""" +"""TaskFlow decorator for :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator`.""" from __future__ import annotations @@ -27,8 +27,8 @@ determine_kwargs, task_decorator_factory, ) -from airflow.providers.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.dataquality.rules import RuleSet +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import RuleSet from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION if TYPE_CHECKING: @@ -89,7 +89,7 @@ def dq_check_task( ``ruleset`` may be passed as a decorator argument when known at Dag-parse time, or returned by the decorated function at runtime. ``table`` or ``asset`` are passed through ``kwargs`` - exactly as on :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator`. The + exactly as on :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator`. The function itself is optional plumbing: return ``None`` to run the check as declared, or a ruleset to use for this run. diff --git a/providers/dataquality/src/airflow/providers/dataquality/engines/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py similarity index 90% rename from providers/dataquality/src/airflow/providers/dataquality/engines/__init__.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py index 70d25099b75cd..cf6a86506a954 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/engines/__init__.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py @@ -16,6 +16,6 @@ # under the License. from __future__ import annotations -from airflow.providers.dataquality.engines.sql import Observation, SQLDQEngine +from airflow.providers.common.dataquality.engines.sql import Observation, SQLDQEngine __all__ = ["Observation", "SQLDQEngine"] diff --git a/providers/dataquality/src/airflow/providers/dataquality/engines/sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py similarity index 96% rename from providers/dataquality/src/airflow/providers/dataquality/engines/sql.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py index ed21549eee450..d9e22ce776258 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/engines/sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py @@ -23,12 +23,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from airflow.providers.dataquality.rules import CUSTOM_SQL_CHECK -from airflow.providers.dataquality.rules.checks import CHECK_SPECS +from airflow.providers.common.dataquality.rules import CUSTOM_SQL_CHECK +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS if TYPE_CHECKING: + from airflow.providers.common.dataquality.rules import DQRule, RuleSet from airflow.providers.common.sql.hooks.sql import DbApiHook - from airflow.providers.dataquality.rules import DQRule, RuleSet log = logging.getLogger(__name__) diff --git a/providers/dataquality/src/airflow/providers/dataquality/example_dags/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/example_dags/__init__.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py diff --git a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py similarity index 95% rename from providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py index c48b39b8611c7..8b2dcb32f9d13 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_custom_sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py @@ -30,9 +30,9 @@ from datetime import datetime from pathlib import Path +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.dataquality.rules import DQRule, RuleSet, Severity from airflow.sdk import DAG DAG_ID = "example_dq_check_custom_sql" diff --git a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_decorator_dynamic.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py similarity index 97% rename from providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_decorator_dynamic.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py index 542875a64cb36..41e194218775f 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_check_decorator_dynamic.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py @@ -29,8 +29,8 @@ from datetime import datetime from pathlib import Path +from airflow.providers.common.dataquality.rules import DQRule, RuleSet from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dataquality.rules import DQRule, RuleSet from airflow.sdk import DAG, task DAG_ID = "example_dq_check_decorator_dynamic" diff --git a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_llm_generated_ruleset.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py similarity index 98% rename from providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_llm_generated_ruleset.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py index c3a8ea8f767ab..15676d32a76fc 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_llm_generated_ruleset.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py @@ -38,8 +38,8 @@ from datetime import datetime from pathlib import Path +from airflow.providers.common.dataquality.rules import RuleSet from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dataquality.rules import RuleSet from airflow.sdk import DAG, task try: diff --git a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_require_quality.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py similarity index 87% rename from providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_require_quality.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py index 539940c422172..af721b10b89da 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_require_quality.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py @@ -20,11 +20,11 @@ Two Dags, wired together only through the ``dq_gated_orders`` asset: - ``example_dq_require_quality_producer`` checks a table and produces the asset. - :class:`~airflow.providers.dataquality.operators.dq_check.DQCheckOperator` attaches its summary + :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` attaches its summary (including the quality ``score``) to the asset event automatically because the asset carries - quality config attached with :func:`~airflow.providers.dataquality.assets.asset_quality`. + quality config attached with :func:`~airflow.providers.common.dataquality.assets.asset_quality`. - ``example_dq_require_quality_consumer`` is scheduled by that asset. Its first task, built by - :func:`~airflow.providers.dataquality.assets.require_quality`, reads the score off the triggering + :func:`~airflow.providers.common.dataquality.assets.require_quality`, reads the score off the triggering asset event and short-circuits -- skipping every downstream task -- when it's missing or below ``min_score``. Downstream tasks never see a run triggered by a bad batch of data. """ @@ -35,10 +35,10 @@ from datetime import datetime from pathlib import Path +from airflow.providers.common.dataquality.assets import asset_quality, require_quality +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dataquality.assets import asset_quality, require_quality -from airflow.providers.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.dataquality.rules import DQRule, RuleSet from airflow.sdk import DAG, Asset, task CONN_ID = "sqlite_default" diff --git a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py similarity index 89% rename from providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py index 077f44e67ae2d..79ca15ae3826c 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/example_dags/example_dq_ruleset_from_yaml.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py @@ -17,10 +17,10 @@ """ Example DAG: load a ruleset from a YAML file instead of declaring it in Python. -A path string is accepted anywhere a :class:`~airflow.providers.dataquality.rules.RuleSet` is -- +A path string is accepted anywhere a :class:`~airflow.providers.common.dataquality.rules.RuleSet` is -- ``DQCheckOperator(ruleset=...)``, ``@task.dq_check(ruleset=...)``, and -:func:`~airflow.providers.dataquality.assets.asset_quality` all resolve it via -:meth:`~airflow.providers.dataquality.rules.RuleSet.from_file` at Dag-parse time. This keeps rules +:func:`~airflow.providers.common.dataquality.assets.asset_quality` all resolve it via +:meth:`~airflow.providers.common.dataquality.rules.RuleSet.from_file` at Dag-parse time. This keeps rules editable by people who don't write Python -- a data steward can change ``orders_ruleset.yaml`` without touching the Dag file. """ @@ -31,8 +31,8 @@ from datetime import datetime from pathlib import Path +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dataquality.operators.dq_check import DQCheckOperator from airflow.sdk import DAG DAG_ID = "example_dq_ruleset_from_yaml" diff --git a/providers/dataquality/src/airflow/providers/dataquality/example_dags/orders_ruleset.yaml b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/example_dags/orders_ruleset.yaml rename to providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml diff --git a/providers/dataquality/src/airflow/providers/dataquality/exceptions.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/exceptions.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py index 961705e59b515..d6be8d160587e 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py @@ -24,6 +24,47 @@ def get_provider_info(): return { "package-name": "apache-airflow-providers-common-dataquality", - "name": "Common Data Quality", - "description": "Common Data Quality Provider\n", + "name": "Data Quality", + "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", + "integrations": [ + { + "integration-name": "Data Quality", + "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/", + "how-to-guide": ["/docs/apache-airflow-providers-common-dataquality/operators.rst"], + "tags": ["software"], + } + ], + "operators": [ + { + "integration-name": "Data Quality", + "python-modules": ["airflow.providers.common.dataquality.operators.dq_check"], + } + ], + "task-decorators": [ + { + "class-name": "airflow.providers.common.dataquality.decorators.dq_check.dq_check_task", + "name": "dq_check", + } + ], + "config": { + "common.dataquality": { + "description": "Configuration for the Data Quality provider results store.\n", + "options": { + "results_path": { + "description": "Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality\nresults are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``.\nWhen unset, checks still run but no history is persisted.\n", + "version_added": "0.1.0", + "type": "string", + "example": "s3://data-platform/airflow-dq", + "default": None, + }, + "results_conn_id": { + "description": "Optional Airflow connection used to access ``results_path``.\n", + "version_added": "0.1.0", + "type": "string", + "example": "aws_default", + "default": None, + }, + }, + } + }, } diff --git a/providers/dataquality/src/airflow/providers/dataquality/operators/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py similarity index 91% rename from providers/dataquality/src/airflow/providers/dataquality/operators/__init__.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py index 73c89bc03920f..048c2ff752049 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/operators/__init__.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py @@ -16,6 +16,6 @@ # under the License. from __future__ import annotations -from airflow.providers.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator __all__ = ["DQCheckOperator"] diff --git a/providers/dataquality/src/airflow/providers/dataquality/operators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py similarity index 90% rename from providers/dataquality/src/airflow/providers/dataquality/operators/dq_check.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py index fb6cc52a281d2..ebc5369e4e323 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/operators/dq_check.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py @@ -21,19 +21,27 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, cast +from airflow.providers.common.dataquality.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config +from airflow.providers.common.dataquality.backends import get_backend_from_config +from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError +from airflow.providers.common.dataquality.results import ( + ERROR, + FAIL, + PASS, + WARN, + DQRun, + RuleResult, + build_summary, +) +from airflow.providers.common.dataquality.rules import RuleSet, describe_rule from airflow.providers.common.sql.operators.sql import BaseSQLOperator -from airflow.providers.dataquality.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config -from airflow.providers.dataquality.backends import get_backend_from_config -from airflow.providers.dataquality.engines.sql import SQLDQEngine -from airflow.providers.dataquality.exceptions import DQCheckFailedError -from airflow.providers.dataquality.results import ERROR, FAIL, PASS, WARN, DQRun, RuleResult, build_summary -from airflow.providers.dataquality.rules import RuleSet, describe_rule from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION if TYPE_CHECKING: + from airflow.providers.common.dataquality.engines.sql import Observation + from airflow.providers.common.dataquality.rules.rule import Condition, Dimension from airflow.providers.common.sql.hooks.sql import DbApiHook - from airflow.providers.dataquality.engines.sql import Observation - from airflow.providers.dataquality.rules.rule import Condition, Dimension from airflow.sdk import Asset, Context log = logging.getLogger(__name__) @@ -49,11 +57,11 @@ class DQCheckOperator(BaseSQLOperator): (a check query failing) always fail the task; rule failures fail the task according to ``fail_on``. - :param ruleset: A :class:`~airflow.providers.dataquality.rules.RuleSet`, its dict form, or a path + :param ruleset: A :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path to a YAML ruleset file. Optional when ``asset`` carries one. :param table: The table to run checks against. Optional when ``asset`` carries one (falling back to the asset name). - :param asset: An asset decorated with :func:`~airflow.providers.dataquality.assets.asset_quality`. + :param asset: An asset decorated with :func:`~airflow.providers.common.dataquality.assets.asset_quality`. Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's outlets so its asset events carry the check summary. @@ -65,7 +73,7 @@ class DQCheckOperator(BaseSQLOperator): :param conn_id: Connection to the database to check (any ``DbApiHook``-compatible type). :param database: Optional database/schema overriding the connection's default. - Results are persisted to the backend configured under ``[dataquality] results_path``; when that's + Results are persisted to the backend configured under ``[common.dataquality] results_path``; when that's unset, checks still run but no history is persisted. There is no per-operator override -- all checks in a deployment share one results store. """ @@ -197,7 +205,7 @@ def _outlet_asset_names(self) -> list[str]: def _persist(self, run: DQRun, results: list[RuleResult]) -> None: backend = get_backend_from_config() if backend is None: - self.log.info("No [dataquality] results_path configured; skipping results persistence") + self.log.info("No [common.dataquality] results_path configured; skipping results persistence") return # Persistence is best-effort: an unreachable results store leaves a gap in # history but must not change the outcome of the check itself. diff --git a/providers/dataquality/src/airflow/providers/dataquality/results.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/results.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/results.py diff --git a/providers/dataquality/src/airflow/providers/dataquality/rules/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py similarity index 87% rename from providers/dataquality/src/airflow/providers/dataquality/rules/__init__.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py index 208dc2d83da2e..e50c9a99f43b9 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/rules/__init__.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py @@ -16,8 +16,8 @@ # under the License. from __future__ import annotations -from airflow.providers.dataquality.rules.checks import CHECK_SPECS, CheckSpec, Dimension -from airflow.providers.dataquality.rules.rule import ( +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS, CheckSpec, Dimension +from airflow.providers.common.dataquality.rules.rule import ( CUSTOM_SQL_CHECK, Condition, DQRule, diff --git a/providers/dataquality/src/airflow/providers/dataquality/rules/checks.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/rules/checks.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py diff --git a/providers/dataquality/src/airflow/providers/dataquality/rules/rule.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py similarity index 98% rename from providers/dataquality/src/airflow/providers/dataquality/rules/rule.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py index 13d5fbf360870..f65a4d67be33e 100644 --- a/providers/dataquality/src/airflow/providers/dataquality/rules/rule.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py @@ -26,8 +26,8 @@ import yaml from pydantic import BaseModel, ConfigDict, Field, model_validator -from airflow.providers.dataquality.exceptions import DQRuleValidationError -from airflow.providers.dataquality.rules.checks import CHECK_SPECS, Dimension +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS, Dimension CUSTOM_SQL_CHECK = "custom_sql" diff --git a/providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/SKILL.md b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/SKILL.md rename to providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md diff --git a/providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json rename to providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json diff --git a/providers/dataquality/src/airflow/providers/dataquality/version_compat.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py similarity index 100% rename from providers/dataquality/src/airflow/providers/dataquality/version_compat.py rename to providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py diff --git a/providers/dataquality/tests/system/dataquality/example_dq_check.py b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py similarity index 96% rename from providers/dataquality/tests/system/dataquality/example_dq_check.py rename to providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py index 16a58f2cf3325..cfd434c8ab431 100644 --- a/providers/dataquality/tests/system/dataquality/example_dq_check.py +++ b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py @@ -18,7 +18,7 @@ Example DAG for the Data Quality provider's ``DQCheckOperator``. Runs against the ``sqlite_default`` connection (Airflow's default local backend, no extra -infrastructure required). Results are persisted through the ``[dataquality] results_path`` config +infrastructure required). Results are persisted through the ``[common.dataquality] results_path`` config option, seeded here via ``AIRFLOW__DQ__RESULTS_PATH`` so the history can be inspected afterwards under ``/tmp/airflow_dq_example/results`` -- a real deployment would instead set ``results_path`` in ``airflow.cfg`` (or its env var) once, for every check to share. @@ -54,10 +54,10 @@ from datetime import datetime from pathlib import Path +from airflow.providers.common.dataquality.assets import asset_quality +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator -from airflow.providers.dataquality.assets import asset_quality -from airflow.providers.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.dataquality.rules import DQRule, RuleSet, Severity from airflow.sdk import DAG, Asset, task DAG_ID = "example_dq_check" @@ -66,7 +66,7 @@ RESULTS_PATH = Path("/tmp/airflow_dq_example/results") # DQCheckOperator has no per-operator results-store override; every check in a deployment -# shares the one store configured under [dataquality] results_path. setdefault() so a real deployment's +# shares the one store configured under [common.dataquality] results_path. setdefault() so a real deployment's # own config is never overridden. os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") diff --git a/providers/dataquality/tests/system/dataquality/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py similarity index 100% rename from providers/dataquality/tests/system/dataquality/__init__.py rename to providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py diff --git a/providers/dataquality/tests/unit/dataquality/backends/test_object_storage.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py similarity index 96% rename from providers/dataquality/tests/unit/dataquality/backends/test_object_storage.py rename to providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py index 88187f5bd2100..e3743e6859b56 100644 --- a/providers/dataquality/tests/unit/dataquality/backends/test_object_storage.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py @@ -21,10 +21,10 @@ import pytest -from airflow.providers.dataquality.backends import get_backend_from_config -from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend -from airflow.providers.dataquality.results import DQRun, RuleResult -from airflow.providers.dataquality.rules import Severity +from airflow.providers.common.dataquality.backends import get_backend_from_config +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.results import DQRun, RuleResult +from airflow.providers.common.dataquality.rules import Severity from tests_common.test_utils.config import conf_vars @@ -354,11 +354,11 @@ def test_read_task_rule_history_cursor_keeps_same_timestamp_records(self, backen class TestGetBackendFromConfig: - @conf_vars({("dataquality", "results_path"): None}) + @conf_vars({("common.dataquality", "results_path"): None}) def test_no_results_path_returns_none(self): assert get_backend_from_config() is None def test_results_path_builds_object_storage_backend(self, tmp_path): - with conf_vars({("dataquality", "results_path"): f"file://{tmp_path}"}): + with conf_vars({("common.dataquality", "results_path"): f"file://{tmp_path}"}): backend = get_backend_from_config() assert isinstance(backend, ObjectStorageResultsBackend) diff --git a/providers/dataquality/tests/unit/dataquality/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py similarity index 100% rename from providers/dataquality/tests/unit/dataquality/__init__.py rename to providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py diff --git a/providers/dataquality/tests/unit/dataquality/decorators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py similarity index 92% rename from providers/dataquality/tests/unit/dataquality/decorators/test_dq_check.py rename to providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py index 1c03e60cfbc01..ecd65c42d01c2 100644 --- a/providers/dataquality/tests/unit/dataquality/decorators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py @@ -20,10 +20,10 @@ import pytest +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.decorators.dq_check import _DQCheckDecoratedOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet from airflow.providers.common.sql.hooks.sql import DbApiHook -from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend -from airflow.providers.dataquality.decorators.dq_check import _DQCheckDecoratedOperator -from airflow.providers.dataquality.rules import DQRule, RuleSet from airflow.sdk.execution_time.context import OutletEventAccessors NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) @@ -47,7 +47,8 @@ def results_backend(tmp_path): """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") with mock.patch( - "airflow.providers.dataquality.operators.dq_check.get_backend_from_config", return_value=backend + "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + return_value=backend, ): yield backend diff --git a/providers/dataquality/tests/unit/dataquality/backends/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py similarity index 100% rename from providers/dataquality/tests/unit/dataquality/backends/__init__.py rename to providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py diff --git a/providers/dataquality/tests/unit/dataquality/engines/test_sql.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py similarity index 97% rename from providers/dataquality/tests/unit/dataquality/engines/test_sql.py rename to providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py index 2b5716673da07..699fc46ecaef9 100644 --- a/providers/dataquality/tests/unit/dataquality/engines/test_sql.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py @@ -20,9 +20,9 @@ import pytest +from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.rules import DQRule, RuleSet from airflow.providers.common.sql.hooks.sql import DbApiHook -from airflow.providers.dataquality.engines.sql import SQLDQEngine -from airflow.providers.dataquality.rules import DQRule, RuleSet @pytest.fixture diff --git a/providers/dataquality/tests/unit/dataquality/decorators/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py similarity index 100% rename from providers/dataquality/tests/unit/dataquality/decorators/__init__.py rename to providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py diff --git a/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py similarity index 93% rename from providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py rename to providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py index 005082bce0f2e..dee1709da10fe 100644 --- a/providers/dataquality/tests/unit/dataquality/operators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -20,12 +20,12 @@ import pytest +from airflow.providers.common.dataquality.assets import asset_quality +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity from airflow.providers.common.sql.hooks.sql import DbApiHook -from airflow.providers.dataquality.assets import asset_quality -from airflow.providers.dataquality.backends.object_storage import ObjectStorageResultsBackend -from airflow.providers.dataquality.exceptions import DQCheckFailedError -from airflow.providers.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.dataquality.rules import DQRule, RuleSet, Severity from airflow.sdk import Asset from airflow.sdk.execution_time.context import OutletEventAccessors @@ -63,7 +63,8 @@ def results_backend(tmp_path): """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") with mock.patch( - "airflow.providers.dataquality.operators.dq_check.get_backend_from_config", return_value=backend + "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + return_value=backend, ): yield backend @@ -220,7 +221,8 @@ def test_backend_failure_does_not_fail_the_check(self, mock_get_db_hook): mock_get_db_hook.return_value = hook with mock.patch( - "airflow.providers.dataquality.operators.dq_check.get_backend_from_config", return_value=backend + "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + return_value=backend, ): summary = operator.execute(make_context()) diff --git a/providers/dataquality/tests/unit/dataquality/engines/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py similarity index 100% rename from providers/dataquality/tests/unit/dataquality/engines/__init__.py rename to providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py diff --git a/providers/dataquality/tests/unit/dataquality/rules/test_checks.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py similarity index 94% rename from providers/dataquality/tests/unit/dataquality/rules/test_checks.py rename to providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py index 42a10fdd37331..a06ae0ef38e02 100644 --- a/providers/dataquality/tests/unit/dataquality/rules/test_checks.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from airflow.providers.dataquality.rules import CHECK_SPECS +from airflow.providers.common.dataquality.rules import CHECK_SPECS class TestCheckSpecs: diff --git a/providers/dataquality/tests/unit/dataquality/rules/test_rule.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py similarity index 97% rename from providers/dataquality/tests/unit/dataquality/rules/test_rule.py rename to providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py index 598317804e471..8165e19ccaec7 100644 --- a/providers/dataquality/tests/unit/dataquality/rules/test_rule.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py @@ -22,8 +22,8 @@ import pytest from pydantic import ValidationError -import airflow.providers.dataquality -from airflow.providers.dataquality.rules import Condition, DQRule, RuleSet, Severity, describe_rule +import airflow.providers.common.dataquality +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity, describe_rule def test_skill_reference_schema_matches_live_model(): @@ -34,7 +34,7 @@ def test_skill_reference_schema_matches_live_model(): newline) if this fails after an intentional schema change. """ schema_path = ( - Path(airflow.providers.dataquality.__file__).parent + Path(airflow.providers.common.dataquality.__file__).parent / "skills" / "dataquality-rule-authoring" / "references" diff --git a/providers/dataquality/tests/unit/dataquality/test_assets.py b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py similarity index 96% rename from providers/dataquality/tests/unit/dataquality/test_assets.py rename to providers/common/dataquality/tests/unit/common/dataquality/test_assets.py index 4056f8ff288a3..3c9f0e8935897 100644 --- a/providers/dataquality/tests/unit/dataquality/test_assets.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py @@ -18,7 +18,7 @@ import pytest -from airflow.providers.dataquality.assets import ( +from airflow.providers.common.dataquality.assets import ( DQ_EXTRA_KEY, DQ_RESULT_EXTRA_KEY, _quality_score_passes, @@ -27,8 +27,8 @@ get_asset_ruleset, require_quality, ) -from airflow.providers.dataquality.exceptions import DQRuleValidationError -from airflow.providers.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules import DQRule, RuleSet from airflow.sdk import Asset from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult from airflow.sdk.execution_time.context import TriggeringAssetEventsAccessor diff --git a/providers/dataquality/tests/unit/dataquality/test_results.py b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py similarity index 96% rename from providers/dataquality/tests/unit/dataquality/test_results.py rename to providers/common/dataquality/tests/unit/common/dataquality/test_results.py index be43b18184fb4..37147de4416cb 100644 --- a/providers/dataquality/tests/unit/dataquality/test_results.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py @@ -18,7 +18,7 @@ import pytest -from airflow.providers.dataquality.results import DQRun, RuleResult, build_summary, compute_score +from airflow.providers.common.dataquality.results import DQRun, RuleResult, build_summary, compute_score def make_result(status: str, name: str = "r") -> RuleResult: diff --git a/providers/dataquality/.gitignore b/providers/dataquality/.gitignore deleted file mode 100644 index f454b41e73c0a..0000000000000 --- a/providers/dataquality/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Do not commit hash files, this is just to speed-up local builds -www-hash.txt -*.iml diff --git a/providers/dataquality/LICENSE b/providers/dataquality/LICENSE deleted file mode 100644 index 11069edd79019..0000000000000 --- a/providers/dataquality/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/providers/dataquality/NOTICE b/providers/dataquality/NOTICE deleted file mode 100644 index a51bd9390d030..0000000000000 --- a/providers/dataquality/NOTICE +++ /dev/null @@ -1,5 +0,0 @@ -Apache Airflow -Copyright 2016-2026 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/dataquality/README.rst b/providers/dataquality/README.rst deleted file mode 100644 index dd2f97d1a3083..0000000000000 --- a/providers/dataquality/README.rst +++ /dev/null @@ -1,52 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Package ``apache-airflow-providers-dataquality`` - -Release: ``0.1.0`` - -``Data Quality Provider`` - -Declarative data quality rules with durable, per-rule execution history. Checks run through -``common.sql`` DB-API hooks; results are persisted to a configurable results store (object -storage or local files) so task, run, and rule-level quality can be inspected over time. - -Provider package ----------------- - -This package is for the ``dataquality`` provider. -All classes for this package are included in the ``airflow.providers.dataquality`` python package. - -Installation ------------- - -You can install this package on top of an existing Airflow installation via -``pip install apache-airflow-providers-dataquality``. For the minimum Airflow version supported, -see ``Requirements`` below. - -Requirements ------------- - -========================================== ================== -PIP package Version required -========================================== ================== -``apache-airflow`` ``>=3.0.0`` -``apache-airflow-providers-common-compat`` ``>=1.15.0`` -``apache-airflow-providers-common-sql`` ``>=2.0.0`` -``pydantic`` ``>=2.11.0`` -``pyyaml`` ``>=6.0.2`` -========================================== ================== diff --git a/providers/dataquality/docs/changelog.rst b/providers/dataquality/docs/changelog.rst deleted file mode 100644 index f4e0951e5d169..0000000000000 --- a/providers/dataquality/docs/changelog.rst +++ /dev/null @@ -1,33 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog" header when there - are some breaking changes and you want to add an explanation to the users on how they - are supposed to deal with them. The changelog is updated and maintained semi-automatically - by release manager. - -``apache-airflow-providers-dataquality`` - - -Changelog ---------- - -0.1.0 -..... - -Initial version of the provider. diff --git a/providers/dataquality/docs/commits.rst b/providers/dataquality/docs/commits.rst deleted file mode 100644 index d8d1bff12fc4a..0000000000000 --- a/providers/dataquality/docs/commits.rst +++ /dev/null @@ -1,35 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! - - .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE - `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - - .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! - -Package apache-airflow-providers-dataquality ------------------------------------------------------- - -``Data Quality Provider`` - - -This is detailed commit list of changes for versions provider package: ``dataquality``. -For high-level changelog, see :doc:`package information including changelog `. - -.. airflow-providers-commits:: diff --git a/providers/dataquality/docs/conf.py b/providers/dataquality/docs/conf.py deleted file mode 100644 index 311cddd302c1e..0000000000000 --- a/providers/dataquality/docs/conf.py +++ /dev/null @@ -1,27 +0,0 @@ -# Disable Flake8 because of all the sphinx imports -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Configuration of Providers docs building.""" - -from __future__ import annotations - -import os - -os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-dataquality" - -from docs.provider_conf import * # noqa: F403 diff --git a/providers/dataquality/docs/index.rst b/providers/dataquality/docs/index.rst deleted file mode 100644 index 2ebe1f1deb4a9..0000000000000 --- a/providers/dataquality/docs/index.rst +++ /dev/null @@ -1,175 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -``apache-airflow-providers-dataquality`` -======================================== - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Basics - - Home - Changelog - Security - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Guides - - Rules, rule sets, and checks - Operators - Decorators - Assets and quality gating - Generating rules with an LLM - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: References - - Configuration - Python API <_api/airflow/providers/dataquality/index> - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Resources - - PyPI Repository - Installing from sources - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Commits - - Detailed list of commits - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: System tests - - System Tests <_api/tests/system/dataquality/index> - - -apache-airflow-providers-dataquality package --------------------------------------------- - -``Data Quality Provider`` - -Declarative data quality rules with durable, per-rule execution history. Checks run through -``common.sql`` DB-API hooks; results are persisted to a configurable results store (object -storage or local files) so task, run, and rule-level quality can be inspected over time. - -See :doc:`rules` for the built-in check catalog (and its cross-database caveats), -:doc:`operators`/:doc:`decorators` for running checks, and :doc:`assets` for attaching rules to -an asset and gating a downstream Dag on its quality score. - -Release: 0.1.0 - -Provider package ----------------- - -This package is for the ``dataquality`` provider. -All classes for this package are included in the ``airflow.providers.dataquality`` python package. - -Installation ------------- - -You can install this package on top of an existing Airflow installation via -``pip install apache-airflow-providers-dataquality``. For the minimum Airflow version supported, -see ``Requirements`` below. - -Requirements ------------- - -The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. - -========================================== ================== -PIP package Version required -========================================== ================== -``apache-airflow`` ``>=3.0.0`` -``apache-airflow-providers-common-compat`` ``>=1.15.0`` -``apache-airflow-providers-common-sql`` ``>=2.0.0`` -``pydantic`` ``>=2.11.0`` -``pyyaml`` ``>=6.0.2`` -========================================== ================== - -.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Commits - - Detailed list of commits - - -apache-airflow-providers-dataquality package ------------------------------------------------------- - -``Data Quality Provider`` - -Declarative data quality rules with durable, per-rule execution history. -Checks run through ``common.sql`` DB-API hooks; results are persisted to a -configurable results store (object storage or local files) so task, run, -and rule-level quality can be inspected over time. - - -Release: 0.1.0 - -Provider package ----------------- - -This package is for the ``dataquality`` provider. -All classes for this package are included in the ``airflow.providers.dataquality`` python package. - -Installation ------------- - -You can install this package on top of an existing Airflow installation via -``pip install apache-airflow-providers-dataquality``. -For the minimum Airflow version supported, see ``Requirements`` below. - -Requirements ------------- - -The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. - -========================================== ================== -PIP package Version required -========================================== ================== -``apache-airflow`` ``>=3.0.0`` -``apache-airflow-providers-common-compat`` ``>=1.15.0`` -``apache-airflow-providers-common-sql`` ``>=2.0.0`` -``pydantic`` ``>=2.11.0`` -``pyyaml`` ``>=6.0.2`` -========================================== ================== - -Downloading official packages ------------------------------ - -You can download officially released packages and verify their checksums and signatures from the -`Official Apache Download site `_ - -* `The apache-airflow-providers-dataquality 0.1.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-dataquality 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/dataquality/docs/installing-providers-from-sources.rst b/providers/dataquality/docs/installing-providers-from-sources.rst deleted file mode 100644 index a72b45ffaa6e8..0000000000000 --- a/providers/dataquality/docs/installing-providers-from-sources.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: /../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/dataquality/docs/security.rst b/providers/dataquality/docs/security.rst deleted file mode 100644 index 15a0ebbb2d054..0000000000000 --- a/providers/dataquality/docs/security.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: /../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/dataquality/provider.yaml b/providers/dataquality/provider.yaml deleted file mode 100644 index c788e673fd8de..0000000000000 --- a/providers/dataquality/provider.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -package-name: apache-airflow-providers-dataquality -name: Data Quality -description: | - ``Data Quality Provider`` - - Declarative data quality rules with durable, per-rule execution history. - Checks run through ``common.sql`` DB-API hooks; results are persisted to a - configurable results store (object storage or local files) so task, run, - and rule-level quality can be inspected over time. - -state: ready -lifecycle: incubation -source-date-epoch: 1751587200 -build-system: flit_core - -# Note that those versions are maintained by release manager - do not update them manually -# with the exception of case where other provider in sources has >= new provider version. -# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have -# to be done in the same PR -versions: - - 0.1.0 - -integrations: - - integration-name: Data Quality - external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-dataquality/ - how-to-guide: - - /docs/apache-airflow-providers-dataquality/operators.rst - tags: [software] - -operators: - - integration-name: Data Quality - python-modules: - - airflow.providers.dataquality.operators.dq_check - -task-decorators: - - class-name: airflow.providers.dataquality.decorators.dq_check.dq_check_task - name: dq_check - -config: - dataquality: - description: | - Configuration for the Data Quality provider results store. - options: - results_path: - description: | - Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality - results are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``. - When unset, checks still run but no history is persisted. - version_added: 0.1.0 - type: string - example: s3://data-platform/airflow-dq - default: ~ - results_conn_id: - description: | - Optional Airflow connection used to access ``results_path``. - version_added: 0.1.0 - type: string - example: aws_default - default: ~ diff --git a/providers/dataquality/pyproject.toml b/providers/dataquality/pyproject.toml deleted file mode 100644 index 10cd4887598f5..0000000000000 --- a/providers/dataquality/pyproject.toml +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! - -# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE -# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -[build-system] -requires = ["flit_core==3.12.0"] -build-backend = "flit_core.buildapi" - -[project] -name = "apache-airflow-providers-dataquality" -version = "0.1.0" -description = "Provider package apache-airflow-providers-dataquality for Apache Airflow" -readme = "README.rst" -license = "Apache-2.0" -license-files = ['LICENSE', 'NOTICE'] -authors = [ - {name="Apache Software Foundation", email="dev@airflow.apache.org"}, -] -maintainers = [ - {name="Apache Software Foundation", email="dev@airflow.apache.org"}, -] -keywords = [ "airflow-provider", "dataquality", "airflow", "integration" ] -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "Framework :: Apache Airflow", - "Framework :: Apache Airflow :: Provider", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: System :: Monitoring", -] -requires-python = ">=3.10" - -# The dependencies should be modified in place in the generated file. -# Any change in the dependencies is preserved when the file is regenerated -# Make sure to run ``prek update-providers-dependencies --all-files`` -# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` -dependencies = [ - "apache-airflow>=3.0.0", - "apache-airflow-providers-common-compat>=1.15.0", - "apache-airflow-providers-common-sql>=2.0.0", - "pydantic>=2.11.0", - "pyyaml>=6.0.2", -] - -[dependency-groups] -dev = [ - "apache-airflow", - "apache-airflow-task-sdk", - "apache-airflow-devel-common", - "apache-airflow-providers-common-compat", - "apache-airflow-providers-common-sql", - # Additional devel dependencies (do not remove this line and add extra development dependencies) -] - -# To build docs: -# -# uv run --group docs build-docs -# -# To enable auto-refreshing build with server: -# -# uv run --group docs build-docs --autobuild -# -# To see more options: -# -# uv run --group docs build-docs --help -# -docs = [ - "apache-airflow-devel-common[docs]" -] - -[tool.uv.sources] -# These names must match the names as defined in the pyproject.toml of the workspace items, -# *not* the workspace folder paths -apache-airflow = {workspace = true} -apache-airflow-devel-common = {workspace = true} -apache-airflow-task-sdk = {workspace = true} -apache-airflow-providers-common-sql = {workspace = true} -apache-airflow-providers-standard = {workspace = true} - -[project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dataquality/0.1.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dataquality/0.1.0/changelog.html" -"Bug Tracker" = "https://github.com/apache/airflow/issues" -"Source Code" = "https://github.com/apache/airflow" -"Slack Chat" = "https://s.apache.org/airflow-slack" -"Mastodon" = "https://fosstodon.org/@airflow" -"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" - -[project.entry-points."apache_airflow_provider"] -provider_info = "airflow.providers.dataquality.get_provider_info:get_provider_info" - -[tool.flit.module] -name = "airflow.providers.dataquality" - -# Explicit sdist contents so the build does not rely on VCS information -# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). -[tool.flit.sdist] -include = [ - "docs/", - "provider.yaml", - "src/airflow/__init__.py", - "src/airflow/providers/__init__.py", - "tests/", -] diff --git a/providers/dataquality/src/airflow/__init__.py b/providers/dataquality/src/airflow/__init__.py deleted file mode 100644 index 5966d6b1d5261..0000000000000 --- a/providers/dataquality/src/airflow/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dataquality/src/airflow/providers/__init__.py b/providers/dataquality/src/airflow/providers/__init__.py deleted file mode 100644 index 5966d6b1d5261..0000000000000 --- a/providers/dataquality/src/airflow/providers/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dataquality/src/airflow/providers/dataquality/__init__.py b/providers/dataquality/src/airflow/providers/dataquality/__init__.py deleted file mode 100644 index 57e75da020e0b..0000000000000 --- a/providers/dataquality/src/airflow/providers/dataquality/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE -# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. -# -# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE -# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -# -from __future__ import annotations - -import packaging.version - -from airflow import __version__ as airflow_version - -__all__ = ["__version__"] - -__version__ = "0.1.0" - -if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "3.0.0" -): - raise RuntimeError( - f"The package `apache-airflow-providers-dataquality:{__version__}` needs Apache Airflow 3.0.0+" - ) diff --git a/providers/dataquality/src/airflow/providers/dataquality/get_provider_info.py b/providers/dataquality/src/airflow/providers/dataquality/get_provider_info.py deleted file mode 100644 index 2f945b1e286db..0000000000000 --- a/providers/dataquality/src/airflow/providers/dataquality/get_provider_info.py +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! -# -# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE -# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - - -def get_provider_info(): - return { - "package-name": "apache-airflow-providers-dataquality", - "name": "Data Quality", - "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", - "integrations": [ - { - "integration-name": "Data Quality", - "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-dataquality/", - "how-to-guide": ["/docs/apache-airflow-providers-dataquality/operators.rst"], - "tags": ["software"], - } - ], - "operators": [ - { - "integration-name": "Data Quality", - "python-modules": ["airflow.providers.dataquality.operators.dq_check"], - } - ], - "task-decorators": [ - { - "class-name": "airflow.providers.dataquality.decorators.dq_check.dq_check_task", - "name": "dq_check", - } - ], - "config": { - "dataquality": { - "description": "Configuration for the Data Quality provider results store.\n", - "options": { - "results_path": { - "description": "Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality\nresults are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``.\nWhen unset, checks still run but no history is persisted.\n", - "version_added": "0.1.0", - "type": "string", - "example": "s3://data-platform/airflow-dq", - "default": None, - }, - "results_conn_id": { - "description": "Optional Airflow connection used to access ``results_path``.\n", - "version_added": "0.1.0", - "type": "string", - "example": "aws_default", - "default": None, - }, - }, - } - }, - } diff --git a/providers/dataquality/tests/conftest.py b/providers/dataquality/tests/conftest.py deleted file mode 100644 index f56ccce0a3f69..0000000000000 --- a/providers/dataquality/tests/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/dataquality/tests/system/__init__.py b/providers/dataquality/tests/system/__init__.py deleted file mode 100644 index 5966d6b1d5261..0000000000000 --- a/providers/dataquality/tests/system/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dataquality/tests/unit/__init__.py b/providers/dataquality/tests/unit/__init__.py deleted file mode 100644 index 5966d6b1d5261..0000000000000 --- a/providers/dataquality/tests/unit/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dataquality/tests/unit/dataquality/operators/__init__.py b/providers/dataquality/tests/unit/dataquality/operators/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/providers/dataquality/tests/unit/dataquality/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/providers/dataquality/tests/unit/dataquality/rules/__init__.py b/providers/dataquality/tests/unit/dataquality/rules/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/providers/dataquality/tests/unit/dataquality/rules/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/pyproject.toml b/pyproject.toml index 58e62bd0a542a..87440e9ad9d30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,9 +209,6 @@ apache-airflow = "airflow.__main__:main" "datadog" = [ "apache-airflow-providers-datadog>=3.8.0" ] -"dataquality" = [ - "apache-airflow-providers-dataquality>=0.1.0" -] "dbt.cloud" = [ "apache-airflow-providers-dbt-cloud>=3.11.0" ] @@ -453,7 +450,6 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-common-sql>=1.18.0", "apache-airflow-providers-databricks>=6.11.0", "apache-airflow-providers-datadog>=3.8.0", - "apache-airflow-providers-dataquality>=0.1.0", "apache-airflow-providers-dbt-cloud>=3.11.0", "apache-airflow-providers-dingding>=3.7.0", "apache-airflow-providers-discord>=3.9.0", @@ -1168,8 +1164,6 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/databricks/tests", "$MYPY_CONFIG_FILE_DIR/providers/datadog/src", "$MYPY_CONFIG_FILE_DIR/providers/datadog/tests", - "$MYPY_CONFIG_FILE_DIR/providers/dataquality/src", - "$MYPY_CONFIG_FILE_DIR/providers/dataquality/tests", "$MYPY_CONFIG_FILE_DIR/providers/dbt/cloud/src", "$MYPY_CONFIG_FILE_DIR/providers/dbt/cloud/tests", "$MYPY_CONFIG_FILE_DIR/providers/dingding/src", @@ -1478,7 +1472,6 @@ apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false apache-airflow-providers-databricks = false apache-airflow-providers-datadog = false -apache-airflow-providers-dataquality = false apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false @@ -1631,7 +1624,6 @@ apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false apache-airflow-providers-databricks = false apache-airflow-providers-datadog = false -apache-airflow-providers-dataquality = false apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false @@ -1795,7 +1787,6 @@ apache-airflow-providers-common-messaging = { workspace = true } apache-airflow-providers-common-sql = { workspace = true } apache-airflow-providers-databricks = { workspace = true } apache-airflow-providers-datadog = { workspace = true } -apache-airflow-providers-dataquality = { workspace = true } apache-airflow-providers-dbt-cloud = { workspace = true } apache-airflow-providers-dingding = { workspace = true } apache-airflow-providers-discord = { workspace = true } @@ -1937,7 +1928,6 @@ members = [ "providers/common/sql", "providers/databricks", "providers/datadog", - "providers/dataquality", "providers/dbt/cloud", "providers/dingding", "providers/discord", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index 8578bd0488803..edb644a179a19 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -64,7 +64,6 @@ services: - ../../../empty:/opt/airflow/providers/common/sql/src - ../../../empty:/opt/airflow/providers/databricks/src - ../../../empty:/opt/airflow/providers/datadog/src - - ../../../empty:/opt/airflow/providers/dataquality/src - ../../../empty:/opt/airflow/providers/dbt/cloud/src - ../../../empty:/opt/airflow/providers/dingding/src - ../../../empty:/opt/airflow/providers/discord/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index 1b48534a58c1f..eb700548346da 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -77,7 +77,6 @@ services: - ../../../providers/common/sql/tests:/opt/airflow/providers/common/sql/tests - ../../../providers/databricks/tests:/opt/airflow/providers/databricks/tests - ../../../providers/datadog/tests:/opt/airflow/providers/datadog/tests - - ../../../providers/dataquality/tests:/opt/airflow/providers/dataquality/tests - ../../../providers/dbt/cloud/tests:/opt/airflow/providers/dbt/cloud/tests - ../../../providers/dingding/tests:/opt/airflow/providers/dingding/tests - ../../../providers/discord/tests:/opt/airflow/providers/discord/tests diff --git a/scripts/ci/prek/generate_dataquality_ruleset_schema.py b/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py similarity index 89% rename from scripts/ci/prek/generate_dataquality_ruleset_schema.py rename to scripts/ci/prek/generate_common_dataquality_ruleset_schema.py index ad404c31016de..dab079ae734d9 100755 --- a/scripts/ci/prek/generate_dataquality_ruleset_schema.py +++ b/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py @@ -17,7 +17,7 @@ # under the License. """ Regenerate the Data Quality RuleSet JSON schema snapshot at -``providers/dataquality/src/airflow/providers/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json``. +``providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json``. """ from __future__ import annotations @@ -27,11 +27,12 @@ from common_prek_utils import AIRFLOW_ROOT_PATH, console -DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "dataquality" +DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "common" / "dataquality" SCHEMA_PATH = DQ_PROVIDER_PATH.joinpath( "src", "airflow", "providers", + "common", "dataquality", "skills", "dataquality-rule-authoring", @@ -43,7 +44,7 @@ import json import sys -from airflow.providers.dataquality.rules import RuleSet +from airflow.providers.common.dataquality.rules import RuleSet sys.stdout.write(json.dumps(RuleSet.model_json_schema(), indent=2)) sys.stdout.write("\n") diff --git a/uv.lock b/uv.lock index 7753fd1df7ef6..92f7b0b418e03 100644 --- a/uv.lock +++ b/uv.lock @@ -53,7 +53,6 @@ apache-airflow-providers-telegram = false apache-airflow-providers-celery = false apache-airflow-providers-docker = false apache-airflow-providers-sendgrid = false -apache-airflow-providers-dataquality = false apache-airflow-providers-common-ai = false apache-airflow = false apache-airflow-shared-observability = false @@ -212,7 +211,6 @@ members = [ "apache-airflow-providers-common-sql", "apache-airflow-providers-databricks", "apache-airflow-providers-datadog", - "apache-airflow-providers-dataquality", "apache-airflow-providers-dbt-cloud", "apache-airflow-providers-dingding", "apache-airflow-providers-discord", @@ -1040,7 +1038,6 @@ all = [ { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-databricks" }, { name = "apache-airflow-providers-datadog" }, - { name = "apache-airflow-providers-dataquality" }, { name = "apache-airflow-providers-dbt-cloud" }, { name = "apache-airflow-providers-dingding" }, { name = "apache-airflow-providers-discord" }, @@ -1240,9 +1237,6 @@ databricks = [ datadog = [ { name = "apache-airflow-providers-datadog" }, ] -dataquality = [ - { name = "apache-airflow-providers-dataquality" }, -] dbt-cloud = [ { name = "apache-airflow-providers-dbt-cloud" }, ] @@ -1654,8 +1648,6 @@ requires-dist = [ { name = "apache-airflow-providers-databricks", marker = "extra == 'databricks'", editable = "providers/databricks" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'all'", editable = "providers/datadog" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'datadog'", editable = "providers/datadog" }, - { name = "apache-airflow-providers-dataquality", marker = "extra == 'all'", editable = "providers/dataquality" }, - { name = "apache-airflow-providers-dataquality", marker = "extra == 'dataquality'", editable = "providers/dataquality" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'all'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'dbt-cloud'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dingding", marker = "extra == 'all'", editable = "providers/dingding" }, @@ -1802,7 +1794,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.28" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-dataquality", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dataquality", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-dataquality", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -4590,28 +4582,16 @@ version = "0.1.0" source = { editable = "providers/common/dataquality" } dependencies = [ { name = "apache-airflow" }, -] - -[package.dev-dependencies] -dev = [ - { name = "apache-airflow" }, - { name = "apache-airflow-devel-common" }, - { name = "apache-airflow-task-sdk" }, -] -docs = [ - { name = "apache-airflow-devel-common", extra = ["docs"] }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, ] [package.metadata] -requires-dist = [{ name = "apache-airflow", editable = "." }] - -[package.metadata.requires-dev] -dev = [ +requires-dist = [ { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-devel-common", editable = "devel-common" }, - { name = "apache-airflow-task-sdk", editable = "task-sdk" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, ] -docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] [[package]] name = "apache-airflow-providers-common-io" @@ -4952,49 +4932,6 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] -[[package]] -name = "apache-airflow-providers-dataquality" -version = "0.1.0" -source = { editable = "providers/dataquality" } -dependencies = [ - { name = "apache-airflow" }, - { name = "apache-airflow-providers-common-compat" }, - { name = "apache-airflow-providers-common-sql" }, - { name = "pydantic" }, - { name = "pyyaml" }, -] - -[package.dev-dependencies] -dev = [ - { name = "apache-airflow" }, - { name = "apache-airflow-devel-common" }, - { name = "apache-airflow-providers-common-compat" }, - { name = "apache-airflow-providers-common-sql" }, - { name = "apache-airflow-task-sdk" }, -] -docs = [ - { name = "apache-airflow-devel-common", extra = ["docs"] }, -] - -[package.metadata] -requires-dist = [ - { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, - { name = "pydantic", specifier = ">=2.11.0" }, - { name = "pyyaml", specifier = ">=6.0.2" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-devel-common", editable = "devel-common" }, - { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, - { name = "apache-airflow-task-sdk", editable = "task-sdk" }, -] -docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] - [[package]] name = "apache-airflow-providers-dbt-cloud" version = "4.9.2" From 9c7c039d94ec9efa0abc084caf5fdf500816661e Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Sat, 11 Jul 2026 01:10:12 +0100 Subject: [PATCH 10/19] Update tests --- providers/common/dataquality/docs/rules.rst | 3 +- .../common/dataquality/rules/rule.py | 30 ++++++++++------ .../dataquality-rule-authoring/SKILL.md | 3 +- .../references/ruleset.schema.json | 2 +- .../common/dataquality/rules/test_rule.py | 36 ++++++++++++++++--- 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/providers/common/dataquality/docs/rules.rst b/providers/common/dataquality/docs/rules.rst index 9ee6b8fd0a254..09a25fa3e9e99 100644 --- a/providers/common/dataquality/docs/rules.rst +++ b/providers/common/dataquality/docs/rules.rst @@ -136,8 +136,7 @@ value, using the same grammar as the ``common.sql`` check operators: - ``equal_to`` -- exact match. Cannot be combined with any other comparison. - ``greater_than`` / ``geq_to`` -- lower bound, exclusive/inclusive. - ``less_than`` / ``leq_to`` -- upper bound, exclusive/inclusive. -- ``tolerance`` -- a percentage that widens ``equal_to`` into a range; only valid together - with ``equal_to``. +- ``tolerance`` -- a percentage that widens the comparison bounds. ``greater_than``/``less_than``/``geq_to``/``leq_to`` may be combined to express a range (e.g. ``{"geq_to": 0, "leq_to": 10}``). diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py index f65a4d67be33e..3fb2c98374996 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py @@ -45,7 +45,7 @@ class Condition(BaseModel): Uses the same grammar as the ``common.sql`` check operators: ``equal_to``, ``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage - ``tolerance`` that widens ``equal_to`` into a range. + ``tolerance`` that widens comparisons. """ @@ -72,8 +72,6 @@ def _validate_comparisons(self) -> Condition: raise ValueError(f"Condition needs at least one comparison out of: {', '.join(comparisons)}") if self.equal_to is not None and len(set_comparisons) > 1: raise ValueError("equal_to cannot be combined with other comparisons") - if self.tolerance is not None and self.equal_to is None: - raise ValueError("tolerance is only supported together with equal_to") return self @classmethod @@ -97,15 +95,27 @@ def evaluate(self, observed: Any) -> bool: value = float(observed) if self.equal_to is not None: if self.tolerance is not None: - low = self.equal_to * (1 - self.tolerance) - high = self.equal_to * (1 + self.tolerance) - return min(low, high) <= value <= max(low, high) + delta = abs(self.equal_to) * self.tolerance + return self.equal_to - delta <= value <= self.equal_to + delta return value == self.equal_to + geq_to = self.geq_to + greater_than = self.greater_than + leq_to = self.leq_to + less_than = self.less_than + if self.tolerance is not None: + if geq_to is not None: + geq_to -= abs(geq_to) * self.tolerance + if greater_than is not None: + greater_than -= abs(greater_than) * self.tolerance + if leq_to is not None: + leq_to += abs(leq_to) * self.tolerance + if less_than is not None: + less_than += abs(less_than) * self.tolerance return ( - (self.greater_than is None or value > self.greater_than) - and (self.less_than is None or value < self.less_than) - and (self.geq_to is None or value >= self.geq_to) - and (self.leq_to is None or value <= self.leq_to) + (greater_than is None or value > greater_than) + and (less_than is None or value < less_than) + and (geq_to is None or value >= geq_to) + and (leq_to is None or value <= leq_to) ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md index 97dc9762b7fa2..8fc005feb7c35 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md @@ -72,8 +72,7 @@ be written as `custom_sql`. - `equal_to` -- exact match. Cannot be combined with any other key below. - `greater_than` / `geq_to` -- lower bound, exclusive / inclusive. - `less_than` / `leq_to` -- upper bound, exclusive / inclusive. -- `tolerance` -- a fraction (e.g. `0.1` for 10%) that widens `equal_to` into a range. Only valid - together with `equal_to`. +- `tolerance` -- a fraction (e.g. `0.1` for 10%) that widens the comparison bounds. Combine `greater_than`/`less_than`/`geq_to`/`leq_to` freely to express a range, e.g. `{"geq_to": 0, "leq_to": 100}`. Never combine `equal_to` with another comparison key (other than diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json index a8f4387981d6d..d807c9cc294b6 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json @@ -2,7 +2,7 @@ "$defs": { "Condition": { "additionalProperties": false, - "description": "Pass/fail condition evaluated against a rule's observed value.\n\nUses the same grammar as the ``common.sql`` check operators: ``equal_to``,\n``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage\n``tolerance`` that widens ``equal_to`` into a range.", + "description": "Pass/fail condition evaluated against a rule's observed value.\n\nUses the same grammar as the ``common.sql`` check operators: ``equal_to``,\n``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage\n``tolerance`` that widens comparisons.", "properties": { "equal_to": { "anyOf": [ diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py index 8165e19ccaec7..15ed5eaae8811 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py @@ -48,18 +48,47 @@ class TestCondition: @pytest.mark.parametrize( ("condition", "observed", "expected"), [ + ({"equal_to": 0}, None, False), ({"equal_to": 0}, 0, True), ({"equal_to": 0}, 1, False), ({"equal_to": 100, "tolerance": 0.1}, 105, True), ({"equal_to": 100, "tolerance": 0.1}, 111, False), + ({"equal_to": -100, "tolerance": 0.1}, -95, True), + ({"equal_to": -100, "tolerance": 0.1}, -105, True), + ({"equal_to": -100, "tolerance": 0.1}, -89, False), + ({"equal_to": -100, "tolerance": 0.1}, -111, False), + ({"geq_to": 10, "tolerance": 0.1}, 9, True), + ({"geq_to": 10, "tolerance": 0.1}, 8.9, False), + ({"geq_to": -1000, "tolerance": 0.1}, -1000, True), + ({"geq_to": -1000, "tolerance": 0.1}, -1100, True), + ({"geq_to": -1000, "tolerance": 0.1}, -1101, False), ({"greater_than": 5}, 6, True), ({"greater_than": 5}, 5, False), - ({"less_than": 5}, 4, True), - ({"geq_to": 5}, 5, True), + ({"greater_than": 10, "tolerance": 0.1}, 9.1, True), + ({"greater_than": 10, "tolerance": 0.1}, 9, False), + ({"greater_than": -100, "tolerance": 0.1}, -109.9, True), + ({"greater_than": -100, "tolerance": 0.1}, -110, False), ({"leq_to": 5}, 6, False), + ({"leq_to": 10, "tolerance": 0.1}, 11, True), + ({"leq_to": 10, "tolerance": 0.1}, 11.1, False), + ({"leq_to": -10, "tolerance": 0.1}, -10, True), + ({"leq_to": -10, "tolerance": 0.1}, -9, True), + ({"leq_to": -10, "tolerance": 0.1}, -8.9, False), + ({"less_than": 5}, 4, True), + ({"less_than": 10, "tolerance": 0.1}, 10.9, True), + ({"less_than": 10, "tolerance": 0.1}, 11, False), + ({"less_than": -100, "tolerance": 0.1}, -90.1, True), + ({"less_than": -100, "tolerance": 0.1}, -90, False), ({"geq_to": 0, "leq_to": 10}, 5, True), ({"geq_to": 0, "leq_to": 10}, 11, False), - ({"equal_to": 0}, None, False), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 9, True), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 22, True), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 8.9, False), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 22.1, False), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -22, True), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -9, True), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -22.1, False), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -8.9, False), ], ) def test_evaluate(self, condition, observed, expected): @@ -71,7 +100,6 @@ def test_evaluate(self, condition, observed, expected): {}, {"tolerance": 0.1}, {"equal_to": 1, "greater_than": 0}, - {"greater_than": 0, "tolerance": 0.1}, {"nonsense": 1}, ], ) From 7c7f02c00a2af29dfc7dc9fedad5052e09446f50 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Mon, 13 Jul 2026 12:25:06 +0100 Subject: [PATCH 11/19] Resolve copilot comments --- .../unit/always/test_project_structure.py | 1 - .../dataquality/backends/object_storage.py | 2 ++ .../example_dq_check_custom_sql.py | 2 +- .../example_dq_check_decorator_dynamic.py | 2 +- .../example_dq_llm_generated_ruleset.py | 2 +- .../example_dq_require_quality.py | 2 +- .../example_dq_ruleset_from_yaml.py | 2 +- .../common/dataquality/operators/dq_check.py | 6 ++++- .../common/dataquality/example_dq_check.py | 4 +-- .../backends/test_object_storage.py | 19 +++++++++++++ .../dataquality/operators/test_dq_check.py | 14 ++++++++++ .../common/dataquality/test_exceptions.py | 27 +++++++++++++++++++ 12 files changed, 74 insertions(+), 9 deletions(-) create mode 100644 providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py index c5f51a8884111..9286bdb5e41a5 100644 --- a/airflow-core/tests/unit/always/test_project_structure.py +++ b/airflow-core/tests/unit/always/test_project_structure.py @@ -107,7 +107,6 @@ def test_providers_modules_should_have_tests(self): "providers/common/compat/tests/unit/common/compat/standard/test_utils.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_base_provider.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_sqs.py", - "providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py", "providers/edge3/tests/unit/edge3/cli/test_example_extended_sysinfo.py", "providers/edge3/tests/unit/edge3/models/test_edge_job.py", "providers/edge3/tests/unit/edge3/models/test_edge_logs.py", diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py index 16b963548b094..6902974d2a91d 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py @@ -106,6 +106,8 @@ def read_task_runs( if before is not None and cursor >= before: continue runs.append(payload) + if len(runs) > limit: + break if len(runs) > limit: break diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py index 8b2dcb32f9d13..185b415e04fb4 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py @@ -40,7 +40,7 @@ TABLE_NAME = "dq_custom_sql_orders" RESULTS_PATH = Path("/tmp/airflow_dq_example/results") -os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") # [START howto_operator_dq_check_custom_sql] custom_sql_ruleset = RuleSet( diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py index 41e194218775f..c32e02f51e2a0 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py @@ -38,7 +38,7 @@ TABLE_NAME = "dq_dynamic_orders" RESULTS_PATH = Path("/tmp/airflow_dq_example/results") -os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") orders_ruleset = RuleSet( name="orders_dynamic", diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py index 15676d32a76fc..5f4ee75d361bd 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py @@ -55,7 +55,7 @@ # __file__ so the path holds regardless of the Dag processor's working directory. SKILLS_DIR = Path(__file__).parent.parent / "skills" / "dataquality-rule-authoring" -os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") if AgentSkillsToolset is not None: with DAG( diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py index af721b10b89da..93ea66a84e363 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py @@ -45,7 +45,7 @@ TABLE_NAME = "dq_gated_orders" RESULTS_PATH = Path("/tmp/airflow_dq_example/results") -os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") # [START howto_asset_quality] gated_orders_ruleset = RuleSet( diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py index 79ca15ae3826c..a1bee5a2431b9 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py @@ -41,7 +41,7 @@ RESULTS_PATH = Path("/tmp/airflow_dq_example/results") RULESET_FILE = Path(__file__).parent / "orders_ruleset.yaml" -os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") with DAG( dag_id=DAG_ID, diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py index ebc5369e4e323..3e13ede36daba 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py @@ -95,7 +95,9 @@ def __init__( config = get_asset_quality_config(asset) or {} ruleset = ruleset if ruleset is not None else config.get("ruleset") table = table or config.get("table") or asset.name - kwargs.setdefault("conn_id", config.get("conn_id")) + asset_conn_id = config.get("conn_id") + if asset_conn_id is not None: + kwargs.setdefault("conn_id", asset_conn_id) outlets = list(kwargs.get("outlets") or []) if asset not in outlets: outlets.append(asset) @@ -107,6 +109,8 @@ def __init__( raise ValueError("ruleset is required, either directly or via an asset_quality() asset") if not table: raise ValueError("table is required, either directly or via an asset_quality() asset") + if not self.conn_id: + raise ValueError("conn_id is required, either directly or via an asset_quality() asset") self.ruleset = ruleset self.table = table self.asset = asset diff --git a/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py index cfd434c8ab431..564d1d339e57b 100644 --- a/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py +++ b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py @@ -19,7 +19,7 @@ Runs against the ``sqlite_default`` connection (Airflow's default local backend, no extra infrastructure required). Results are persisted through the ``[common.dataquality] results_path`` config -option, seeded here via ``AIRFLOW__DQ__RESULTS_PATH`` so the history can be inspected +option, seeded here via ``AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH`` so the history can be inspected afterwards under ``/tmp/airflow_dq_example/results`` -- a real deployment would instead set ``results_path`` in ``airflow.cfg`` (or its env var) once, for every check to share. @@ -68,7 +68,7 @@ # DQCheckOperator has no per-operator results-store override; every check in a deployment # shares the one store configured under [common.dataquality] results_path. setdefault() so a real deployment's # own config is never overridden. -os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") # [START howto_operator_dq_check_ruleset] orders_ruleset = RuleSet( diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py index e3743e6859b56..cf63c1a37a07d 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py @@ -291,6 +291,25 @@ def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monk # no further — date=2026-07-02/01 must not be read. assert len(read_paths) == 2 + def test_read_task_runs_stops_inside_date_partition_once_limit_is_reached(self, backend, monkeypatch): + for index in range(1, 5): + backend.write_run( + make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), + [make_result()], + ) + + read_paths: list[Any] = [] + original_read_json = backend._read_json + monkeypatch.setattr( + backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=1) + + assert len(result["items"]) == 1 + assert result["next_cursor"] is not None + assert len(read_paths) == 2 + def test_read_task_rule_history_filters_to_task(self, backend): backend.write_run(make_run(run_uid="run1"), [make_result()]) backend.write_run( diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py index dee1709da10fe..d84423696dea6 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -82,6 +82,20 @@ def test_rejects_bad_fail_on(self): with pytest.raises(ValueError, match="fail_on"): DQCheckOperator(task_id="dq", ruleset=RULESET, table="orders", conn_id="c", fail_on="maybe") + def test_requires_conn_id(self): + with pytest.raises(ValueError, match="conn_id is required"): + DQCheckOperator(task_id="dq", ruleset=RULESET, table="orders") + + def test_asset_without_conn_id_fails_fast(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + table="analytics.orders", + ) + + with pytest.raises(ValueError, match="conn_id is required"): + DQCheckOperator(task_id="dq", asset=asset) + def test_asset_supplies_defaults_and_outlet(self): asset = asset_quality( Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py b/providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py new file mode 100644 index 0000000000000..712a8775a0a0c --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError, DQRuleValidationError + + +def test_dq_rule_validation_error_is_value_error(): + assert isinstance(DQRuleValidationError("invalid rule"), ValueError) + + +def test_dq_check_failed_error_is_runtime_error(): + assert isinstance(DQCheckFailedError("failed check"), RuntimeError) From bec9f7c39025194498ba4d992caee3d0ad0c648e Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 14 Jul 2026 00:31:27 +0100 Subject: [PATCH 12/19] Resolve first round of comments from kaxil --- .../providers/common/dataquality/assets.py | 62 +++++++++------ .../common/dataquality/backends/__init__.py | 12 ++- .../dataquality/backends/object_storage.py | 18 +++-- .../common/dataquality/engines/sql.py | 41 ++++++---- .../common/dataquality/rules/rule.py | 33 +++++++- .../dataquality-rule-authoring/SKILL.md | 5 ++ .../references/ruleset.schema.json | 22 ++++-- .../common/dataquality/version_compat.py | 37 --------- .../backends/test_backends_init.py | 38 +++++++++ .../backends/test_object_storage.py | 23 +++++- .../common/dataquality/engines/test_sql.py | 53 ++++++++++++- .../dataquality/operators/test_dq_check.py | 2 + .../common/dataquality/rules/test_rule.py | 68 ++++++++++++++++ .../unit/common/dataquality/test_assets.py | 79 ++++++++++++------- 14 files changed, 367 insertions(+), 126 deletions(-) delete mode 100644 providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py create mode 100644 providers/common/dataquality/tests/unit/common/dataquality/backends/test_backends_init.py diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py index 4655e9478ab89..263cd53924503 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py @@ -104,8 +104,17 @@ def _quality_score_passes( asset: Asset, min_score: float, triggering_asset_events: Mapping[Asset, Sequence[AssetEventDagRunReferenceResult]], + *, + require_all: bool = True, ) -> bool: - """Pure decision logic behind :func:`require_quality`, kept separate so it is testable without a Dag.""" + """ + Pure decision logic behind :func:`require_quality`, kept separate so it is testable without a Dag. + + ``consumed_asset_events`` carries no ordering guarantee, so the triggering event is never + picked positionally. By default every triggering event must pass, since several producer + runs can coalesce into one consumer run; pass ``require_all=False`` to gate on the single + most recent event (by ``timestamp``) instead. + """ events = triggering_asset_events.get(asset, []) if not events: log.warning( @@ -114,23 +123,26 @@ def _quality_score_passes( ) return False - summary = events[-1].extra.get(DQ_RESULT_EXTRA_KEY) - score = summary.get("score") if isinstance(summary, dict) else None - if not isinstance(score, (int, float)) or isinstance(score, bool): - log.warning( - "require_quality(%s): triggering event has no data quality summary; skipping downstream tasks", - asset.name, - ) - return False - - if score < min_score: - log.warning( - "require_quality(%s): score %s below required minimum %s; skipping downstream tasks", - asset.name, - score, - min_score, - ) - return False + events_to_check = events if require_all else [max(events, key=lambda event: event.timestamp)] + + for event in events_to_check: + summary = event.extra.get(DQ_RESULT_EXTRA_KEY) + score = summary.get("score") if isinstance(summary, dict) else None + if not isinstance(score, (int, float)) or isinstance(score, bool): + log.warning( + "require_quality(%s): triggering event has no data quality summary; skipping downstream tasks", + asset.name, + ) + return False + + if score < min_score: + log.warning( + "require_quality(%s): score %s below required minimum %s; skipping downstream tasks", + asset.name, + score, + min_score, + ) + return False return True @@ -140,9 +152,10 @@ def require_quality( *, min_score: float, task_id: str | None = None, + require_all: bool = True, ) -> Any: """ - Gate a Dag run on the data quality score attached to one of its triggering asset events. + Gate a Dag run on the data quality score attached to its triggering asset event(s). Reads the summary a :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dataquality.result"]`` @@ -154,12 +167,15 @@ def require_quality( start = require_quality(orders, min_score=0.95) start >> process_orders() - :param asset: The asset whose triggering event carries the quality summary. + :param asset: The asset whose triggering event(s) carry the quality summary. :param min_score: Minimum required score in ``[0, 1]``; the run proceeds only when the - triggering event's score is at least this value. + checked event(s)' score is at least this value. :param task_id: Task id for the generated gate task. Defaults to ``f"require_quality_{asset.name}"`` so gating on several assets in one Dag doesn't collide on task id. + :param require_all: When ``True`` (default), every asset event that triggered this run must + pass ``min_score`` — the safe default when several producer runs coalesce into one + consumer run. Set to ``False`` to gate on only the most recent triggering event instead. """ if not 0 <= min_score <= 1: raise ValueError(f"min_score must be between 0 and 1, got {min_score!r}") @@ -167,6 +183,8 @@ def require_quality( @task.short_circuit(task_id=gate_task_id) def _require_quality(**context: Any) -> bool: - return _quality_score_passes(asset, min_score, context["triggering_asset_events"]) + return _quality_score_passes( + asset, min_score, context["triggering_asset_events"], require_all=require_all + ) return _require_quality() diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py index a41919c9f627a..6008fc360bd41 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py @@ -16,9 +16,12 @@ # under the License. from __future__ import annotations -from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from typing import TYPE_CHECKING -__all__ = ["ObjectStorageResultsBackend", "get_backend_from_config"] +if TYPE_CHECKING: + from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend + +__all__ = ["get_backend_from_config"] def get_backend_from_config() -> ObjectStorageResultsBackend | None: @@ -28,5 +31,10 @@ def get_backend_from_config() -> ObjectStorageResultsBackend | None: results_path = conf.get("common.dataquality", "results_path", fallback=None) if not results_path: return None + + # Imported lazily: ObjectStoragePath pulls in fsspec/upath, which shouldn't be paid at + # Dag-parse time (this module is imported eagerly by the operator) when results_path unset. + from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend + conn_id = conf.get("common.dataquality", "results_conn_id", fallback=None) return ObjectStorageResultsBackend(results_path=results_path, conn_id=conn_id or None) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py index 6902974d2a91d..c640d24abe61c 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py @@ -19,7 +19,7 @@ Each DQ check writes a keyed JSON document plus read indexes optimized for the UI: - runs/by_task/dag_id=/task_id=/date=<2026-07-04>/.json + runs/by_task/dag_id=/task_id=/date=<2026-07-04>/__.json Canonical run record: ``{"run": ..., "results": [...], "summary": ...}``. runs/by_task_instance/dag_id=/task_id=/__.json @@ -53,9 +53,10 @@ def __init__(self, results_path: str, conn_id: str | None = None) -> None: def write_run(self, run: DQRun, results: list[RuleResult]) -> None: timestamp = run.started_at or datetime.now(tz=timezone.utc).isoformat() + compact_ts = self._get_safe_key(timestamp) payload = self._build_run_payload(run, results) - self._write_run_file(run, timestamp[:10], payload) + self._write_run_file(run, timestamp[:10], compact_ts, payload) self._write_task_instance_index(run, payload) self._write_rule_indexes(run, results, timestamp) @@ -86,8 +87,11 @@ def read_task_runs( exhausted without walking to the end. ``date=`` partition names sort correctly as plain strings, so directories are walked - newest-first and scanning stops as soon as ``limit + 1`` matching runs have been - collected — a task with years of history doesn't pay for a full scan on every page. + newest-first. Filenames are ``{started_at}__{run_uid}.json``, so sorting each partition's + filenames before scanning also walks newest-first within the partition -- required for + the early exit below to stop on the actual newest runs rather than an arbitrary subset. + Scanning stops as soon as ``limit + 1`` matching runs have been collected — a task with + years of history doesn't pay for a full scan on every page. """ task_dir = self.root / "runs" / "by_task" / f"dag_id={dag_id}" / f"task_id={task_id}" if not task_dir.exists(): @@ -96,7 +100,7 @@ def read_task_runs( date_dirs = sorted((path for path in task_dir.iterdir() if path.is_dir()), reverse=True) runs = [] for date_dir in date_dirs: - for path in date_dir.iterdir(): + for path in sorted(date_dir.iterdir(), key=lambda p: p.name, reverse=True): if not path.name.endswith(".json"): continue payload = self._read_json(path) @@ -130,7 +134,7 @@ def read_by_task_instance( ) return self._read_json_or_raise(path) - def _write_run_file(self, run: DQRun, date_part: str, payload: dict[str, Any]) -> None: + def _write_run_file(self, run: DQRun, date_part: str, compact_ts: str, payload: dict[str, Any]) -> None: run_dir = ( self.root / "runs" @@ -140,7 +144,7 @@ def _write_run_file(self, run: DQRun, date_part: str, payload: dict[str, Any]) - / f"date={date_part}" ) run_dir.mkdir(parents=True, exist_ok=True) - (run_dir / f"{run.run_uid}.json").write_text(json.dumps(payload, default=str)) + (run_dir / f"{compact_ts}__{run.run_uid}.json").write_text(json.dumps(payload, default=str)) def _write_task_instance_index(self, run: DQRun, payload: dict[str, Any]) -> None: ti_dir = self.root / "runs" / "by_task_instance" / f"dag_id={run.dag_id}" / f"task_id={run.task_id}" diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py index d9e22ce776258..eb1f101c468b1 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py @@ -83,23 +83,21 @@ def build_rule_sql(self, rule: DQRule, table: str, partition_clause: str | None def _measure_builtin( self, rules: list[DQRule], table: str, partition_clause: str | None ) -> list[Observation]: - sql = self.build_batch_sql(rules, table, partition_clause) + # Built once per rule and reused below, instead of re-deriving each rule's SQL for the + # batch, for the returned Observation, and again in the per-rule fallback. + rule_sql = {rule.rule_uid: self.build_rule_sql(rule, table, partition_clause) for rule in rules} + sql = " UNION ALL ".join(rule_sql.values()) log.info("Running %d built-in checks against %s", len(rules), table) started = time.monotonic() try: records = self.hook.get_records(sql) - except Exception as e: - elapsed_ms = (time.monotonic() - started) * 1000 - log.exception("Check query failed for table %s", table) - return [ - Observation( - rule=rule, - duration_ms=elapsed_ms, - error_message=str(e), - sql=self.build_rule_sql(rule, table, partition_clause), - ) - for rule in rules - ] + except Exception: + log.exception( + "Batched check query failed for table %s; falling back to one query per rule so a " + "single bad rule doesn't mark every other rule in the batch as errored", + table, + ) + return [self._measure_builtin_single(rule, rule_sql[rule.rule_uid]) for rule in rules] elapsed_ms = (time.monotonic() - started) * 1000 observed_by_uid = {str(row[0]): row[1] for row in records or []} return [ @@ -108,11 +106,26 @@ def _measure_builtin( observed_value=observed_by_uid.get(rule.rule_uid), duration_ms=elapsed_ms, error_message=None if rule.rule_uid in observed_by_uid else "No result returned for rule", - sql=self.build_rule_sql(rule, table, partition_clause), + sql=rule_sql[rule.rule_uid], ) for rule in rules ] + def _measure_builtin_single(self, rule: DQRule, sql: str) -> Observation: + started = time.monotonic() + try: + row = self.hook.get_first(sql) + except Exception as e: + elapsed_ms = (time.monotonic() - started) * 1000 + log.exception("Check query failed for rule %s", rule.name) + return Observation(rule=rule, duration_ms=elapsed_ms, error_message=str(e), sql=sql) + elapsed_ms = (time.monotonic() - started) * 1000 + if row is None: + return Observation( + rule=rule, duration_ms=elapsed_ms, error_message="No result returned for rule", sql=sql + ) + return Observation(rule=rule, observed_value=row[0], duration_ms=elapsed_ms, sql=sql) + def _measure_custom(self, rule: DQRule, table: str) -> Observation: if rule.sql is None: raise ValueError(f"Rule {rule.name!r} has no SQL to execute") diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py index 3fb2c98374996..ba501ee78ff09 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py @@ -136,9 +136,14 @@ class DQRule(BaseModel): May reference the target table as ``{table}``. :param severity: ``error`` (fails the task by default) or ``warn`` (recorded only). :param partition_clause: Extra predicate ANDed into the check's WHERE clause. - :param previous_name: Set when renaming a rule, to keep its history continuous. + :param previous_name: Set when renaming a rule, to keep its history continuous. A rule whose + ``previous_name`` happens to match another rule's identity (same check/column/condition) + collides on ``rule_uid``; set an explicit ``id`` on one of them to avoid this. :param description: Human-readable description shown in results and the UI. When omitted, the provider generates a short default description from the rule and condition. + :param id: Explicit, stable identity for this rule's history. When set, it is used verbatim + as the ``rule_uid`` instead of the derived hash, so it survives ``previous_name`` chains + and sidesteps any collision between rules that would otherwise hash the same. Invalid input raises pydantic's own :class:`~pydantic.ValidationError`. """ @@ -147,12 +152,19 @@ class DQRule(BaseModel): name: str check: str = Field(description="One of the built-in checks, or custom_sql.") - condition: Condition | dict[str, Any] | None = None + condition: Condition | None = None column: str | None = None sql: str | None = None severity: Severity = Severity.ERROR partition_clause: str | None = None previous_name: str | None = None + id: str | None = Field( + default=None, + description=( + "Explicit, stable identity for this rule's history, used verbatim as rule_uid " + "instead of a derived hash." + ), + ) description: str | None = Field( default=None, description=( @@ -197,7 +209,15 @@ def _validate(self) -> DQRule: @property def rule_uid(self) -> str: - """Stable identity across runs: survives severity/dimension tweaks and Dag refactors.""" + """ + Stable identity across runs: survives severity/dimension tweaks and Dag refactors. + + Uses ``id`` verbatim when set. Otherwise derives a hash from the rule's identity -- + set ``id`` explicitly to sidestep a collision between two rules that would otherwise + hash the same (see ``previous_name``). + """ + if self.id: + return self.id condition = cast("Condition", self.condition) identity = { "name": self.previous_name or self.name, @@ -217,7 +237,7 @@ def to_dict(self) -> dict[str, Any]: "condition": condition.to_dict(), "severity": self.severity.value, } - for optional in ("column", "sql", "partition_clause", "previous_name", "description"): + for optional in ("column", "sql", "partition_clause", "previous_name", "id", "description"): value = getattr(self, optional) if value is not None: data[optional] = value @@ -267,6 +287,11 @@ def _validate(self) -> RuleSet: duplicates = {name for name in names if names.count(name) > 1} if duplicates: raise ValueError(f"Duplicate rule names in ruleset {self.name!r}: {sorted(duplicates)}") + uids = [rule.rule_uid for rule in self.rules] + colliding_uids = {uid for uid in uids if uids.count(uid) > 1} + if colliding_uids: + colliding_names = sorted(rule.name for rule in self.rules if rule.rule_uid in colliding_uids) + raise ValueError(f"Rules {colliding_names} in ruleset {self.name!r} collide on rule_uid") return self def to_dict(self) -> dict[str, Any]: diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md index 8fc005feb7c35..d07e74d3f60ab 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md @@ -80,6 +80,11 @@ Combine `greater_than`/`less_than`/`geq_to`/`leq_to` freely to express a range, ## `custom_sql`: when a catalog check doesn't fit +> **Security note:** `custom_sql` executes the given statement verbatim against the configured +> connection. Prefer a built-in check whenever one fits. Treat any model-generated `custom_sql` +> rule as requiring human review before it runs against a production connection, and use +> read-only credentials for data quality connections. + Use `check: "custom_sql"` with a `sql` statement resolving to a single scalar whenever: - The check needs more than one column (e.g. `end_date >= start_date`), a join, or a subquery. diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json index d807c9cc294b6..830b94e81fb17 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json @@ -82,7 +82,7 @@ }, "DQRule": { "additionalProperties": false, - "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", + "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous. A rule whose\n ``previous_name`` happens to match another rule's identity (same check/column/condition)\n collides on ``rule_uid``; set an explicit ``id`` on one of them to avoid this.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n:param id: Explicit, stable identity for this rule's history. When set, it is used verbatim\n as the ``rule_uid`` instead of the derived hash, so it survives ``previous_name`` chains\n and sidesteps any collision between rules that would otherwise hash the same.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", "properties": { "name": { "title": "Name", @@ -98,16 +98,11 @@ { "$ref": "#/$defs/Condition" }, - { - "additionalProperties": true, - "type": "object" - }, { "type": "null" } ], - "default": null, - "title": "Condition" + "default": null }, "column": { "anyOf": [ @@ -161,6 +156,19 @@ "default": null, "title": "Previous Name" }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Explicit, stable identity for this rule's history, used verbatim as rule_uid instead of a derived hash.", + "title": "Id" + }, "description": { "anyOf": [ { diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py deleted file mode 100644 index b47842705282b..0000000000000 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY -# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS -# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT -# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE -# -from __future__ import annotations - - -def get_base_airflow_version_tuple() -> tuple[int, int, int]: - from packaging.version import Version - - from airflow import __version__ - - airflow_version = Version(__version__) - return airflow_version.major, airflow_version.minor, airflow_version.micro - - -AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0) - -__all__ = ["AIRFLOW_V_3_1_PLUS"] diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_backends_init.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_backends_init.py new file mode 100644 index 0000000000000..3b027c1b73950 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_backends_init.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import subprocess +import sys + + +def test_get_backend_from_config_does_not_eagerly_import_object_storage(): + """ + ``ObjectStorageResultsBackend`` pulls in ``ObjectStoragePath`` (fsspec/upath), which + ``get_backend_from_config()`` shouldn't pay for at import time when no ``results_path`` is + configured -- it's only needed once a backend is actually built. Runs in a subprocess for a + clean ``sys.modules`` unaffected by other tests importing the submodule directly. + """ + script = ( + "import sys\n" + "from airflow.providers.common.dataquality.backends import get_backend_from_config\n" + "assert 'airflow.providers.common.dataquality.backends.object_storage' not in sys.modules\n" + "assert get_backend_from_config() is None\n" + "assert 'airflow.providers.common.dataquality.backends.object_storage' not in sys.modules\n" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py index cf63c1a37a07d..c05e7ec221ce3 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py @@ -80,7 +80,7 @@ def test_write_run_stores_keyed_json_payload(self, backend): / "dag_id=orders_pipeline" / "task_id=dq" / "date=2026-07-04" - / "abc123.json" + / "2026-07-04T06_00_00_00_00__abc123.json" ) payload = json.loads(path.read_text()) @@ -306,10 +306,27 @@ def test_read_task_runs_stops_inside_date_partition_once_limit_is_reached(self, result = backend.read_task_runs("orders_pipeline", "dq", limit=1) - assert len(result["items"]) == 1 - assert result["next_cursor"] is not None + assert [r["run"]["run_uid"] for r in result["items"]] == ["run4"] + assert result["next_cursor"] == "2026-07-04T06:00:04+00:00|run4" assert len(read_paths) == 2 + def test_read_task_runs_within_partition_returns_newest_first_regardless_of_write_order(self, backend): + """More than limit+1 runs in one date partition must still surface the true newest ones. + + Filenames are random run_uids with no ordering guarantee from iterdir(), so this only + passes when the partition is sorted by its started_at-prefixed filename before scanning. + """ + for index in (3, 1, 4, 2): + backend.write_run( + make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), + [make_result()], + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=2) + + assert [r["run"]["run_uid"] for r in result["items"]] == ["run4", "run3"] + assert result["next_cursor"] == "2026-07-04T06:00:03+00:00|run3" + def test_read_task_rule_history_filters_to_task(self, backend): backend.write_run(make_run(run_uid="run1"), [make_result()]) backend.write_run( diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py index 699fc46ecaef9..b3185d6a1e89a 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py @@ -77,17 +77,64 @@ def test_builtin_rules_measured_from_single_query(self, hook): assert by_name["volume"].sql == SQLDQEngine(hook).build_rule_sql(VOLUME, "orders") assert all(obs.error_message is None for obs in observations) - def test_query_failure_yields_error_observations(self, hook): + def test_builds_each_rules_sql_exactly_once_on_success(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 42)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + engine = SQLDQEngine(hook) + + with mock.patch.object(engine, "build_rule_sql", wraps=engine.build_rule_sql) as build_rule_sql: + engine.measure(ruleset, "orders") + + assert build_rule_sql.call_count == 2 + + def test_builds_each_rules_sql_exactly_once_on_batch_failure(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.return_value = (0,) + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + engine = SQLDQEngine(hook) + + with mock.patch.object(engine, "build_rule_sql", wraps=engine.build_rule_sql) as build_rule_sql: + engine.measure(ruleset, "orders") + + assert build_rule_sql.call_count == 2 + + def test_batch_failure_falls_back_to_per_rule_queries(self, hook): hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.return_value = (0,) ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) observations = SQLDQEngine(hook).measure(ruleset, "orders") + assert hook.get_first.call_count == 2 assert len(observations) == 2 - assert all(obs.error_message == "connection refused" for obs in observations) - assert all(obs.observed_value is None for obs in observations) + assert all(obs.error_message is None for obs in observations) + assert all(obs.observed_value == 0 for obs in observations) assert all(obs.sql for obs in observations) + def test_batch_failure_fallback_isolates_a_single_bad_rule(self, hook): + """One rule's query failing in the per-rule fallback must not fail the other rules.""" + hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.side_effect = [(0,), RuntimeError("no such column: no_such_column")] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].observed_value == 0 + assert by_name["nulls"].error_message is None + assert by_name["volume"].observed_value is None + assert by_name["volume"].error_message == "no such column: no_such_column" + + def test_per_rule_fallback_query_with_no_rows_is_an_error(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.return_value = None + ruleset = RuleSet(name="s", rules=(NULLS,)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + assert observations[0].error_message == "No result returned for rule" + assert observations[0].observed_value is None + def test_missing_rule_in_result_is_an_error(self, hook): hook.get_records.return_value = [(NULLS.rule_uid, 0)] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py index d84423696dea6..cb3485ca06748 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -208,8 +208,10 @@ def test_fail_on_never_records_failures_without_raising(self, mock_get_db_hook): @mock.patch.object(DQCheckOperator, "get_db_hook") def test_execution_error_always_fails_task(self, mock_get_db_hook): + """A connection that's actually down fails the batch query and every per-rule fallback query.""" operator, hook = make_operator(records=None, fail_on="never") hook.get_records.side_effect = RuntimeError("boom") + hook.get_first.side_effect = RuntimeError("boom") mock_get_db_hook.return_value = hook with pytest.raises(DQCheckFailedError, match="errored"): diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py index 15ed5eaae8811..9ee827c0cb7fc 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py @@ -117,6 +117,18 @@ def test_condition_dict_is_coerced(self): rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) assert isinstance(rule.condition, Condition) + @pytest.mark.parametrize( + "condition", + [ + {"nonsense": 1}, + {"tolerance": 0.1}, + {"equal_to": 1, "greater_than": 0}, + ], + ) + def test_malformed_condition_dict_rejected_at_construction(self, condition): + with pytest.raises(ValidationError): + DQRule(name="r", check="null_count", column="c", condition=condition) + def test_rule_uid_is_stable_across_severity(self): base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) tweaked = DQRule( @@ -155,6 +167,50 @@ def test_previous_name_keeps_uid(self): ) assert renamed.rule_uid == old.rule_uid + def test_previous_name_can_collide_with_another_rules_identity(self): + """ + Documents the known collision this ``rule_uid`` scheme can produce. + + A rule renamed away from ``old_name`` derives the same uid as another, unrelated rule + that is still actually named ``old_name`` with the same check/column/condition. Set an + explicit ``id`` on one of them to avoid this (see ``test_explicit_id_avoids_collision``). + """ + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition={"equal_to": 0}, + previous_name="old_name", + ) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + assert renamed.rule_uid == unrelated.rule_uid + + def test_explicit_id_is_used_verbatim_as_rule_uid(self): + rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}, id="my-stable-id") + assert rule.rule_uid == "my-stable-id" + + def test_explicit_id_avoids_collision(self): + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition={"equal_to": 0}, + previous_name="old_name", + id="renamed-rule", + ) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + assert renamed.rule_uid != unrelated.rule_uid + + def test_explicit_id_survives_condition_and_previous_name_changes(self): + first = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}, id="stable") + tightened = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 5}, id="stable") + assert first.rule_uid == tightened.rule_uid + + def test_id_is_serialized_and_round_trips(self): + rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}, id="my-stable-id") + assert rule.to_dict()["id"] == "my-stable-id" + assert DQRule.from_dict(rule.to_dict()) == rule + @pytest.mark.parametrize( "kwargs", [ @@ -258,6 +314,18 @@ def test_duplicate_rule_names_rejected(self): with pytest.raises(ValidationError, match="Duplicate rule names"): RuleSet(name="s", rules=(rule, rule)) + def test_colliding_rule_uids_rejected(self): + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition={"equal_to": 0}, + previous_name="old_name", + ) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + with pytest.raises(ValidationError, match="collide on rule_uid"): + RuleSet(name="s", rules=(renamed, unrelated)) + def test_from_dict_round_trip(self): ruleset = RuleSet( name="orders", diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py index 3c9f0e8935897..05826bc50d503 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py @@ -41,18 +41,28 @@ ORDERS = Asset("orders") -def make_triggering_events(*, extra: dict, asset: Asset = ORDERS) -> TriggeringAssetEventsAccessor: +def make_event( + *, + extra: dict, + asset: Asset = ORDERS, + run_id: str = "manual__2026-07-04", + timestamp: str = "2026-07-04T06:00:00Z", +) -> AssetEventDagRunReferenceResult: event = { "asset": {"name": asset.name, "uri": asset.uri, "extra": {}}, "extra": extra, "source_task_id": "dq", "source_dag_id": "orders_pipeline", - "source_run_id": "manual__2026-07-04", + "source_run_id": run_id, "source_map_index": -1, "source_aliases": [], - "timestamp": "2026-07-04T06:00:00Z", + "timestamp": timestamp, } - return TriggeringAssetEventsAccessor.build([AssetEventDagRunReferenceResult.model_validate(event)]) + return AssetEventDagRunReferenceResult.model_validate(event) + + +def make_triggering_events(*, extra: dict, asset: Asset = ORDERS) -> TriggeringAssetEventsAccessor: + return TriggeringAssetEventsAccessor.build([make_event(extra=extra, asset=asset)]) class TestAssetQuality: @@ -130,35 +140,50 @@ def test_fails_when_summary_has_no_score(self): events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"passed": 3}}) assert _quality_score_passes(ORDERS, 0.5, events) is False - def test_uses_most_recent_event_when_several_present(self): - first = AssetEventDagRunReferenceResult.model_validate( - { - "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, - "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, - "source_task_id": "dq", - "source_dag_id": "orders_pipeline", - "source_run_id": "run1", - "source_map_index": -1, - "source_aliases": [], - "timestamp": "2026-07-03T06:00:00Z", - } + def test_require_all_fails_when_any_event_is_below_min_score(self): + low = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" ) - second = AssetEventDagRunReferenceResult.model_validate( - { - "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, - "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, - "source_task_id": "dq", - "source_dag_id": "orders_pipeline", - "source_run_id": "run2", - "source_map_index": -1, - "source_aliases": [], - "timestamp": "2026-07-04T06:00:00Z", - } + high = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" + ) + events = TriggeringAssetEventsAccessor.build([low, high]) + + assert _quality_score_passes(ORDERS, 0.9, events) is False + + def test_require_all_passes_when_every_event_meets_min_score(self): + first = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.95}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" + ) + second = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" ) events = TriggeringAssetEventsAccessor.build([first, second]) assert _quality_score_passes(ORDERS, 0.9, events) is True + def test_latest_only_uses_most_recent_event_when_require_all_false(self): + first = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" + ) + second = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events, require_all=False) is True + + def test_latest_only_fails_when_most_recent_event_is_below_min_score(self): + first = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" + ) + second = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events, require_all=False) is False + class TestRequireQuality: def test_rejects_min_score_below_zero(self): From 7b618e1424ad715e1fc4221604f65cd9d34e8964 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 14 Jul 2026 00:55:00 +0100 Subject: [PATCH 13/19] Resolve conflicts --- .github/boring-cyborg.yml | 3 - providers/common/dataquality/docs/index.rst | 72 +++++++++++++++++++ providers/common/dataquality/provider.yaml | 6 +- providers/common/dataquality/pyproject.toml | 11 +++ .../common/dataquality/get_provider_info.py | 6 +- uv.lock | 26 +++++++ 6 files changed, 115 insertions(+), 9 deletions(-) diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 5a8b8179fba9e..cd31dc07f4036 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -144,9 +144,6 @@ labelPRBasedOnFilePath: provider:docker: - providers/docker/** - provider:common-dataquality: - - providers/common/dataquality/** - provider:edge: - providers/edge3/** diff --git a/providers/common/dataquality/docs/index.rst b/providers/common/dataquality/docs/index.rst index e36368b27ad1a..0e26c1be95b17 100644 --- a/providers/common/dataquality/docs/index.rst +++ b/providers/common/dataquality/docs/index.rst @@ -55,6 +55,78 @@ PyPI Repository Installing from sources +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/common/dataquality/index> + + +apache-airflow-providers-common-dataquality package +--------------------------------------------------- + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +See :doc:`rules` for the built-in check catalog (and its cross-database caveats), +:doc:`operators`/:doc:`decorators` for running checks, and :doc:`assets` for attaching rules to +an asset and gating a downstream Dag on its quality score. + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``common.dataquality`` provider. +All classes for this package are included in the ``airflow.providers.common.dataquality`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-common-dataquality``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-common-dataquality package +------------------------------------------------------ + ``Data Quality Provider`` Declarative data quality rules with durable, per-rule execution history. diff --git a/providers/common/dataquality/provider.yaml b/providers/common/dataquality/provider.yaml index 80969a57621e7..ffe1ad33ef588 100644 --- a/providers/common/dataquality/provider.yaml +++ b/providers/common/dataquality/provider.yaml @@ -17,7 +17,7 @@ --- package-name: apache-airflow-providers-common-dataquality -name: Data Quality +name: Common Data Quality description: | ``Data Quality Provider`` @@ -39,14 +39,14 @@ versions: - 0.1.0 integrations: - - integration-name: Data Quality + - integration-name: Common Data Quality external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/ how-to-guide: - /docs/apache-airflow-providers-common-dataquality/operators.rst tags: [software] operators: - - integration-name: Data Quality + - integration-name: Common Data Quality python-modules: - airflow.providers.common.dataquality.operators.dq_check diff --git a/providers/common/dataquality/pyproject.toml b/providers/common/dataquality/pyproject.toml index b8c4e6f6b5dd9..bbf3f9764da68 100644 --- a/providers/common/dataquality/pyproject.toml +++ b/providers/common/dataquality/pyproject.toml @@ -60,6 +60,17 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=3.0.0", + "apache-airflow-providers-common-compat>=1.15.0", + "apache-airflow-providers-common-sql>=2.0.0", + "pydantic>=2.11.0", + "pyyaml>=6.0.2", +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", "apache-airflow-providers-common-compat", "apache-airflow-providers-common-sql", # Additional devel dependencies (do not remove this line and add extra development dependencies) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py index d6be8d160587e..756efc23b8549 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py @@ -24,11 +24,11 @@ def get_provider_info(): return { "package-name": "apache-airflow-providers-common-dataquality", - "name": "Data Quality", + "name": "Common Data Quality", "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", "integrations": [ { - "integration-name": "Data Quality", + "integration-name": "Common Data Quality", "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/", "how-to-guide": ["/docs/apache-airflow-providers-common-dataquality/operators.rst"], "tags": ["software"], @@ -36,7 +36,7 @@ def get_provider_info(): ], "operators": [ { - "integration-name": "Data Quality", + "integration-name": "Common Data Quality", "python-modules": ["airflow.providers.common.dataquality.operators.dq_check"], } ], diff --git a/uv.lock b/uv.lock index 92f7b0b418e03..0983a47d28960 100644 --- a/uv.lock +++ b/uv.lock @@ -4584,6 +4584,20 @@ dependencies = [ { name = "apache-airflow" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql" }, + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, ] [package.metadata] @@ -4591,7 +4605,19 @@ requires-dist = [ { name = "apache-airflow", editable = "." }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "pydantic", specifier = ">=2.11.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, ] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] [[package]] name = "apache-airflow-providers-common-io" From 5dfa93e0ce59116a159b153c5b007f83f5382191 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 14 Jul 2026 01:22:39 +0100 Subject: [PATCH 14/19] Update docs --- providers/common/dataquality/docs/assets.rst | 4 ++++ providers/common/dataquality/docs/rules.rst | 19 +++++++++++++++++++ .../example_dq_require_quality.py | 8 +++++++- .../dataquality-rule-authoring/SKILL.md | 1 + 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/providers/common/dataquality/docs/assets.rst b/providers/common/dataquality/docs/assets.rst index ce88628d334fb..00cb0e7c2444d 100644 --- a/providers/common/dataquality/docs/assets.rst +++ b/providers/common/dataquality/docs/assets.rst @@ -41,6 +41,10 @@ via ``asset=`` (see :doc:`operators`) instead of ``table``/``ruleset``/``conn_id resolves all three from the asset's config, adds the asset to its own outlets, and attaches its summary -- including the quality ``score`` used below -- to the asset event. +Only one Dag should call ``asset_quality()`` for a given asset ``name``/``uri``. Airflow keeps one +shared record per asset across all Dags, so if more than one Dag attaches (or omits) config for the +same asset, whichever Dag parsed most recently determines what's stored. + Gating a consumer Dag on quality ------------------------------------ diff --git a/providers/common/dataquality/docs/rules.rst b/providers/common/dataquality/docs/rules.rst index 09a25fa3e9e99..1e9a7aa00ded7 100644 --- a/providers/common/dataquality/docs/rules.rst +++ b/providers/common/dataquality/docs/rules.rst @@ -63,6 +63,8 @@ can also be proposed by an LLM -- see :doc:`agents`. ``"region = 'EU'"``. Combines with the operator-level ``partition_clause``, if any. - ``previous_name`` -- set when renaming a rule so its execution history stays continuous (see `Identity and history`_). +- ``id`` -- explicit, stable identity for this rule's history, used verbatim as ``rule_uid`` + instead of the derived hash (see `Identity and history`_). - ``description`` -- optional human-readable text shown in data quality results. When omitted, the provider generates a short default description from the rule and condition. - ``dimension`` -- one of ``completeness``, ``uniqueness``, ``validity``, ``freshness``, @@ -218,3 +220,20 @@ Renaming a rule outright would normally start a new history under the new name; column="order_id", condition={"equal_to": 0}, ) + +The derived hash isn't guaranteed unique across every possible ruleset: two rules can end up +with the same ``rule_uid`` if their identity fields happen to line up (for example, a renamed +rule's ``previous_name`` matching another rule's current ``name``, ``check``, ``column``, and +``condition``). ``RuleSet`` validates against this and raises if it happens. If you'd rather +not depend on the hash at all, set ``id`` and it's used as the ``rule_uid`` directly: + +.. code-block:: python + + DQRule( + name="order_id_is_unique", + previous_name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + id="order_id_uniqueness", # stable regardless of future renames or condition tweaks + ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py index 93ea66a84e363..2f2c18230e49f 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py @@ -51,7 +51,13 @@ gated_orders_ruleset = RuleSet( name="gated_orders_quality", rules=( - DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + id="order_id_not_null", # explicit rule_uid, stable across future renames + ), DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), ), diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md index d07e74d3f60ab..cdfd7b7262ae5 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md @@ -43,6 +43,7 @@ unique within the ruleset. | `severity` | no | `"error"` (default) or `"warn"`. `"error"` fails the task on a failing rule (subject to the operator's `fail_on`); `"warn"` only records it. | | `partition_clause` | no | Extra SQL predicate ANDed into this rule's `WHERE` clause, e.g. `"region = 'EU'"`. | | `previous_name` | no | Only set when told a rule is being renamed, to keep its history continuous. | +| `id` | no | Explicit stable identity for this rule's history, used verbatim instead of the derived hash. Only set this when told to, e.g. to avoid a collision between a renamed rule and an unrelated rule that reuses its old name. | | `description` | no | Human-readable text shown in data quality results. Use it when a clear business meaning is known; otherwise omit it and let Airflow generate a default description. | | `dimension` | no | One of `completeness`, `uniqueness`, `validity`, `freshness`, `volume`, `consistency`. Defaults to the check's catalog dimension (`validity` for `custom_sql`) -- only set this explicitly when a `custom_sql` rule measures something the default doesn't capture (e.g. a freshness check written as `custom_sql` should set `"dimension": "freshness"`). Leave it unset for every built-in check. | From cf10db4401cedec371ad52785a9292da20d8e368 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 14 Jul 2026 18:36:40 +0100 Subject: [PATCH 15/19] Fix mypy --- providers/common/dataquality/docs/rules.rst | 2 +- .../example_dq_check_custom_sql.py | 6 +- .../example_dq_check_decorator_dynamic.py | 10 ++- .../example_dq_require_quality.py | 8 +- .../common/dataquality/rules/rule.py | 6 +- .../dataquality-rule-authoring/SKILL.md | 2 +- .../references/ruleset.schema.json | 4 +- .../common/dataquality/example_dq_check.py | 38 +++++----- .../dataquality/decorators/test_dq_check.py | 6 +- .../common/dataquality/engines/test_sql.py | 10 +-- .../dataquality/operators/test_dq_check.py | 8 +- .../common/dataquality/rules/test_rule.py | 76 +++++++++++-------- .../unit/common/dataquality/test_assets.py | 4 +- 13 files changed, 97 insertions(+), 83 deletions(-) diff --git a/providers/common/dataquality/docs/rules.rst b/providers/common/dataquality/docs/rules.rst index 1e9a7aa00ded7..ab59fc7e7f8ab 100644 --- a/providers/common/dataquality/docs/rules.rst +++ b/providers/common/dataquality/docs/rules.rst @@ -63,7 +63,7 @@ can also be proposed by an LLM -- see :doc:`agents`. ``"region = 'EU'"``. Combines with the operator-level ``partition_clause``, if any. - ``previous_name`` -- set when renaming a rule so its execution history stays continuous (see `Identity and history`_). -- ``id`` -- explicit, stable identity for this rule's history, used verbatim as ``rule_uid`` +- ``id`` -- explicit, stable identity for this rule's history, used directly as ``rule_uid`` instead of the derived hash (see `Identity and history`_). - ``description`` -- optional human-readable text shown in data quality results. When omitted, the provider generates a short default description from the rule and condition. diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py index 185b415e04fb4..e0b0086e5e938 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py @@ -31,7 +31,7 @@ from pathlib import Path from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.sdk import DAG @@ -52,7 +52,7 @@ # {table} is substituted by the SQL engine at check time, not an f-string # placeholder -- a cross-column comparison no single-column built-in can express. sql="SELECT COUNT(*) FROM {table} WHERE shipped_at < ordered_at", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), ), DQRule( name="high_value_order_ratio", @@ -60,7 +60,7 @@ sql=( "SELECT CAST(SUM(CASE WHEN amount > 100 THEN 1 ELSE 0 END) AS REAL) / COUNT(*) FROM {table}" ), - condition={"leq_to": 0.5}, + condition=Condition(leq_to=0.5), severity=Severity.WARN, ), ), diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py index c32e02f51e2a0..bc92664b3aa1a 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py @@ -29,7 +29,7 @@ from datetime import datetime from pathlib import Path -from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.sdk import DAG, task @@ -43,8 +43,10 @@ orders_ruleset = RuleSet( name="orders_dynamic", rules=( - DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), - DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + DQRule( + name="order_id_not_null", check="null_count", column="order_id", condition=Condition(equal_to=0) + ), + DQRule(name="row_count_present", check="row_count", condition=Condition(greater_than=0)), ), ) @@ -52,7 +54,7 @@ name="orders_dynamic_strict", rules=( *orders_ruleset.rules, - DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition=Condition(geq_to=0)), ), ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py index 2f2c18230e49f..2b9c93c661489 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py @@ -37,7 +37,7 @@ from airflow.providers.common.dataquality.assets import asset_quality, require_quality from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.sdk import DAG, Asset, task @@ -55,11 +55,11 @@ name="order_id_not_null", check="null_count", column="order_id", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), id="order_id_not_null", # explicit rule_uid, stable across future renames ), - DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), - DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition=Condition(geq_to=0)), + DQRule(name="row_count_present", check="row_count", condition=Condition(greater_than=0)), ), ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py index ba501ee78ff09..6efa97faf423c 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py @@ -141,7 +141,7 @@ class DQRule(BaseModel): collides on ``rule_uid``; set an explicit ``id`` on one of them to avoid this. :param description: Human-readable description shown in results and the UI. When omitted, the provider generates a short default description from the rule and condition. - :param id: Explicit, stable identity for this rule's history. When set, it is used verbatim + :param id: Explicit, stable identity for this rule's history. When set, it is used directly as the ``rule_uid`` instead of the derived hash, so it survives ``previous_name`` chains and sidesteps any collision between rules that would otherwise hash the same. @@ -161,7 +161,7 @@ class DQRule(BaseModel): id: str | None = Field( default=None, description=( - "Explicit, stable identity for this rule's history, used verbatim as rule_uid " + "Explicit, stable identity for this rule's history, used directly as rule_uid " "instead of a derived hash." ), ) @@ -212,7 +212,7 @@ def rule_uid(self) -> str: """ Stable identity across runs: survives severity/dimension tweaks and Dag refactors. - Uses ``id`` verbatim when set. Otherwise derives a hash from the rule's identity -- + Uses ``id`` directly when set. Otherwise derives a hash from the rule's identity -- set ``id`` explicitly to sidestep a collision between two rules that would otherwise hash the same (see ``previous_name``). """ diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md index cdfd7b7262ae5..202d2b650f3a1 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md @@ -43,7 +43,7 @@ unique within the ruleset. | `severity` | no | `"error"` (default) or `"warn"`. `"error"` fails the task on a failing rule (subject to the operator's `fail_on`); `"warn"` only records it. | | `partition_clause` | no | Extra SQL predicate ANDed into this rule's `WHERE` clause, e.g. `"region = 'EU'"`. | | `previous_name` | no | Only set when told a rule is being renamed, to keep its history continuous. | -| `id` | no | Explicit stable identity for this rule's history, used verbatim instead of the derived hash. Only set this when told to, e.g. to avoid a collision between a renamed rule and an unrelated rule that reuses its old name. | +| `id` | no | Explicit stable identity for this rule's history, used directly instead of the derived hash. Only set this when told to, e.g. to avoid a collision between a renamed rule and an unrelated rule that reuses its old name. | | `description` | no | Human-readable text shown in data quality results. Use it when a clear business meaning is known; otherwise omit it and let Airflow generate a default description. | | `dimension` | no | One of `completeness`, `uniqueness`, `validity`, `freshness`, `volume`, `consistency`. Defaults to the check's catalog dimension (`validity` for `custom_sql`) -- only set this explicitly when a `custom_sql` rule measures something the default doesn't capture (e.g. a freshness check written as `custom_sql` should set `"dimension": "freshness"`). Leave it unset for every built-in check. | diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json index 830b94e81fb17..2f56d1867fe99 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json @@ -82,7 +82,7 @@ }, "DQRule": { "additionalProperties": false, - "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous. A rule whose\n ``previous_name`` happens to match another rule's identity (same check/column/condition)\n collides on ``rule_uid``; set an explicit ``id`` on one of them to avoid this.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n:param id: Explicit, stable identity for this rule's history. When set, it is used verbatim\n as the ``rule_uid`` instead of the derived hash, so it survives ``previous_name`` chains\n and sidesteps any collision between rules that would otherwise hash the same.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", + "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous. A rule whose\n ``previous_name`` happens to match another rule's identity (same check/column/condition)\n collides on ``rule_uid``; set an explicit ``id`` on one of them to avoid this.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n:param id: Explicit, stable identity for this rule's history. When set, it is used directly\n as the ``rule_uid`` instead of the derived hash, so it survives ``previous_name`` chains\n and sidesteps any collision between rules that would otherwise hash the same.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", "properties": { "name": { "title": "Name", @@ -166,7 +166,7 @@ } ], "default": null, - "description": "Explicit, stable identity for this rule's history, used verbatim as rule_uid instead of a derived hash.", + "description": "Explicit, stable identity for this rule's history, used directly as rule_uid instead of a derived hash.", "title": "Id" }, "description": { diff --git a/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py index 564d1d339e57b..a2adde9d64395 100644 --- a/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py +++ b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py @@ -56,7 +56,7 @@ from airflow.providers.common.dataquality.assets import asset_quality from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.sdk import DAG, Asset, task @@ -78,89 +78,89 @@ name="order_id_not_null", check="null_count", column="order_id", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), ), DQRule( name="order_id_unique", check="unique_violations", column="order_id", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), ), DQRule( name="customer_id_not_null", check="null_count", column="customer_id", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), severity=Severity.WARN, ), DQRule( name="discount_null_ratio", check="null_ratio", column="discount", - condition={"leq_to": 0.25}, + condition=Condition(leq_to=0.25), severity=Severity.WARN, ), DQRule( name="region_distinct_count", check="distinct_count", column="region", - condition={"geq_to": 2}, + condition=Condition(geq_to=2), severity=Severity.WARN, ), DQRule( name="amount_min_ge_zero", check="min", column="amount", - condition={"geq_to": 0}, + condition=Condition(geq_to=0), ), DQRule( name="amount_max_le_100", check="max", column="amount", - condition={"leq_to": 100}, + condition=Condition(leq_to=100), ), DQRule( name="amount_mean_between_5_and_60", check="mean", column="amount", - condition={"geq_to": 5, "leq_to": 60}, + condition=Condition(geq_to=5, leq_to=60), ), DQRule( name="quantity_min_ge_one", check="min", column="quantity", - condition={"geq_to": 1}, + condition=Condition(geq_to=1), ), DQRule( name="quantity_max_le_ten", check="max", column="quantity", - condition={"leq_to": 10}, + condition=Condition(leq_to=10), ), DQRule( name="amount_non_negative", check="custom_sql", # {table} is substituted by the SQL engine at check time, not an f-string placeholder. sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), ), DQRule( name="high_value_order_count", check="custom_sql", sql="SELECT COUNT(*) FROM {table} WHERE amount > 100", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), severity=Severity.WARN, ), DQRule( name="row_count_present", check="row_count", - condition={"greater_than": 0}, + condition=Condition(greater_than=0), severity=Severity.WARN, ), DQRule( name="row_count_at_least_three", check="row_count", - condition={"geq_to": 3}, + condition=Condition(geq_to=3), ), ), ) @@ -182,24 +182,24 @@ name="strict_order_id_not_null", check="null_count", column="order_id", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), ), DQRule( name="strict_amount_max_le_20", check="max", column="amount", - condition={"leq_to": 20}, + condition=Condition(leq_to=20), ), DQRule( name="strict_row_count_equal_two", check="row_count", - condition={"equal_to": 2}, + condition=Condition(equal_to=2), ), DQRule( name="strict_region_distinct_count", check="distinct_count", column="region", - condition={"geq_to": 4}, + condition=Condition(geq_to=4), severity=Severity.WARN, ), ), diff --git a/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py index ecd65c42d01c2..59b2520034ee8 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py @@ -22,11 +22,11 @@ from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend from airflow.providers.common.dataquality.decorators.dq_check import _DQCheckDecoratedOperator -from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.sdk.execution_time.context import OutletEventAccessors -NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +NULLS = DQRule(name="nulls", check="null_count", column="id", condition=Condition(equal_to=0)) RULESET = RuleSet(name="orders", rules=(NULLS,)) @@ -80,7 +80,7 @@ def test_none_return_runs_check_as_declared(self, mock_get_db_hook): @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") def test_ruleset_can_be_provided_only_at_runtime(self, mock_get_db_hook): - other_rule = DQRule(name="row_count_ok", check="row_count", condition={"greater_than": 0}) + other_rule = DQRule(name="row_count_ok", check="row_count", condition=Condition(greater_than=0)) other_ruleset = RuleSet(name="dynamic", rules=(other_rule,)) operator, hook = make_decorated_operator( diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py index b3185d6a1e89a..eebae09b192ab 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py @@ -21,7 +21,7 @@ import pytest from airflow.providers.common.dataquality.engines.sql import SQLDQEngine -from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet from airflow.providers.common.sql.hooks.sql import DbApiHook @@ -30,13 +30,13 @@ def hook(): return mock.create_autospec(DbApiHook, instance=True) -NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) -VOLUME = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) +NULLS = DQRule(name="nulls", check="null_count", column="id", condition=Condition(equal_to=0)) +VOLUME = DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)) CUSTOM = DQRule( name="negatives", check="custom_sql", sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), ) @@ -55,7 +55,7 @@ def test_partition_clauses_are_anded(self, hook): name="nulls", check="null_count", column="id", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), partition_clause="region = 'EU'", ) sql = SQLDQEngine(hook).build_batch_sql([rule], "orders", "ds = '2026-07-04'") diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py index cb3485ca06748..a0ea1af5080a4 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -24,14 +24,14 @@ from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend from airflow.providers.common.dataquality.exceptions import DQCheckFailedError from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator -from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.sdk import Asset from airflow.sdk.execution_time.context import OutletEventAccessors -NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +NULLS = DQRule(name="nulls", check="null_count", column="id", condition=Condition(equal_to=0)) VOLUME_WARN = DQRule( - name="volume", check="row_count", condition={"greater_than": 100}, severity=Severity.WARN + name="volume", check="row_count", condition=Condition(greater_than=100), severity=Severity.WARN ) RULESET = RuleSet(name="orders", rules=(NULLS, VOLUME_WARN)) @@ -168,7 +168,7 @@ def test_rule_description_is_persisted(self, mock_get_db_hook, results_backend): name="nulls", check="null_count", column="id", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), description="Order IDs must always be present.", ) operator, hook = make_operator( diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py index 9ee827c0cb7fc..50c4ddd13e8e1 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py @@ -114,7 +114,7 @@ def test_to_dict_round_trip(self): class TestDQRule: def test_condition_dict_is_coerced(self): - rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) # type: ignore[arg-type] assert isinstance(rule.condition, Condition) @pytest.mark.parametrize( @@ -130,39 +130,39 @@ def test_malformed_condition_dict_rejected_at_construction(self, condition): DQRule(name="r", check="null_count", column="c", condition=condition) def test_rule_uid_is_stable_across_severity(self): - base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + base = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0)) tweaked = DQRule( name="r", check="null_count", column="c", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), severity=Severity.WARN, ) assert base.rule_uid == tweaked.rule_uid def test_rule_uid_is_stable_across_description(self): - base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + base = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0)) described = DQRule( name="r", check="null_count", column="c", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), description="Column c must be populated.", ) assert base.rule_uid == described.rule_uid def test_rule_uid_changes_with_condition(self): - one = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) - two = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 1}) + one = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0)) + two = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=1)) assert one.rule_uid != two.rule_uid def test_previous_name_keeps_uid(self): - old = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + old = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) renamed = DQRule( name="new_name", check="null_count", column="c", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), previous_name="old_name", ) assert renamed.rule_uid == old.rule_uid @@ -179,14 +179,16 @@ def test_previous_name_can_collide_with_another_rules_identity(self): name="new_name", check="null_count", column="c", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), previous_name="old_name", ) - unrelated = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) assert renamed.rule_uid == unrelated.rule_uid - def test_explicit_id_is_used_verbatim_as_rule_uid(self): - rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}, id="my-stable-id") + def test_explicit_id_is_used_directly_as_rule_uid(self): + rule = DQRule( + name="r", check="null_count", column="c", condition=Condition(equal_to=0), id="my-stable-id" + ) assert rule.rule_uid == "my-stable-id" def test_explicit_id_avoids_collision(self): @@ -194,20 +196,24 @@ def test_explicit_id_avoids_collision(self): name="new_name", check="null_count", column="c", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), previous_name="old_name", id="renamed-rule", ) - unrelated = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) assert renamed.rule_uid != unrelated.rule_uid def test_explicit_id_survives_condition_and_previous_name_changes(self): - first = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}, id="stable") - tightened = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 5}, id="stable") + first = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0), id="stable") + tightened = DQRule( + name="r", check="null_count", column="c", condition=Condition(equal_to=5), id="stable" + ) assert first.rule_uid == tightened.rule_uid def test_id_is_serialized_and_round_trips(self): - rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}, id="my-stable-id") + rule = DQRule( + name="r", check="null_count", column="c", condition=Condition(equal_to=0), id="my-stable-id" + ) assert rule.to_dict()["id"] == "my-stable-id" assert DQRule.from_dict(rule.to_dict()) == rule @@ -227,7 +233,7 @@ def test_invalid_rules_rejected(self, kwargs): DQRule(**kwargs) def test_row_count_needs_no_column(self): - rule = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) + rule = DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)) assert rule.column is None @pytest.mark.parametrize( @@ -244,7 +250,7 @@ def test_row_count_needs_no_column(self): ], ) def test_default_dimension_comes_from_check_catalog(self, check, column, expected_dimension): - rule = DQRule(name="r", check=check, column=column, condition={"equal_to": 0}) + rule = DQRule(name="r", check=check, column=column, condition=Condition(equal_to=0)) assert rule.dimension.value == expected_dimension def test_custom_sql_dimension_can_be_overridden(self): @@ -252,18 +258,24 @@ def test_custom_sql_dimension_can_be_overridden(self): name="freshness_check", check="custom_sql", sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), dimension="freshness", ) assert rule.dimension.value == "freshness" def test_invalid_dimension_rejected(self): with pytest.raises(ValidationError): - DQRule(name="r", check="row_count", condition={"greater_than": 0}, dimension="not_a_dimension") + DQRule( + name="r", check="row_count", condition=Condition(greater_than=0), dimension="not_a_dimension" + ) def test_explicit_dimension_matching_default_is_omitted_from_to_dict(self): rule = DQRule( - name="r", check="null_count", column="c", condition={"equal_to": 0}, dimension="completeness" + name="r", + check="null_count", + column="c", + condition=Condition(equal_to=0), + dimension="completeness", ) assert "dimension" not in rule.to_dict() @@ -272,7 +284,7 @@ def test_dimension_override_round_trips(self): name="freshness_check", check="custom_sql", sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), dimension="freshness", ) assert rule.to_dict()["dimension"] == "freshness" @@ -287,7 +299,7 @@ def test_to_dict_round_trip(self): name="r", check="custom_sql", sql="SELECT COUNT(*) FROM {table} WHERE x < 0", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), severity=Severity.WARN, partition_clause="ds = '2026-07-04'", description="No rows should have negative x.", @@ -298,19 +310,19 @@ def test_description_is_serialized(self): rule = DQRule( name="r", check="row_count", - condition={"greater_than": 0}, + condition=Condition(greater_than=0), description="Orders table should not be empty.", ) assert rule.to_dict()["description"] == "Orders table should not be empty." def test_default_description_uses_column_when_available(self): - rule = DQRule(name="amount_min", check="min", column="amount", condition={"geq_to": 0}) + rule = DQRule(name="amount_min", check="min", column="amount", condition=Condition(geq_to=0)) assert describe_rule(rule) == "amount should be greater than or equal to 0.0" class TestRuleSet: def test_duplicate_rule_names_rejected(self): - rule = DQRule(name="r", check="row_count", condition={"greater_than": 0}) + rule = DQRule(name="r", check="row_count", condition=Condition(greater_than=0)) with pytest.raises(ValidationError, match="Duplicate rule names"): RuleSet(name="s", rules=(rule, rule)) @@ -319,10 +331,10 @@ def test_colliding_rule_uids_rejected(self): name="new_name", check="null_count", column="c", - condition={"equal_to": 0}, + condition=Condition(equal_to=0), previous_name="old_name", ) - unrelated = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) with pytest.raises(ValidationError, match="collide on rule_uid"): RuleSet(name="s", rules=(renamed, unrelated)) @@ -330,8 +342,8 @@ def test_from_dict_round_trip(self): ruleset = RuleSet( name="orders", rules=( - DQRule(name="volume", check="row_count", condition={"greater_than": 0}), - DQRule(name="ids", check="null_count", column="id", condition={"equal_to": 0}), + DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)), + DQRule(name="ids", check="null_count", column="id", condition=Condition(equal_to=0)), ), ) assert RuleSet.from_dict(ruleset.to_dict()) == ruleset diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py index 05826bc50d503..95a3310e50f34 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py @@ -28,14 +28,14 @@ require_quality, ) from airflow.providers.common.dataquality.exceptions import DQRuleValidationError -from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet from airflow.sdk import Asset from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult from airflow.sdk.execution_time.context import TriggeringAssetEventsAccessor RULESET = RuleSet( name="orders", - rules=(DQRule(name="volume", check="row_count", condition={"greater_than": 0}),), + rules=(DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)),), ) ORDERS = Asset("orders") From c1bb26d36628077dfdf95c1f58cf83475b81d1c9 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Tue, 14 Jul 2026 22:59:22 +0100 Subject: [PATCH 16/19] update error message --- .../src/airflow/providers/common/dataquality/engines/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py index eb1f101c468b1..e5c9ac884b0b6 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py @@ -93,10 +93,10 @@ def _measure_builtin( records = self.hook.get_records(sql) except Exception: log.exception( - "Batched check query failed for table %s; falling back to one query per rule so a " - "single bad rule doesn't mark every other rule in the batch as errored", + "Batched check query failed for table %s; falling back to one query execution per rule", table, ) + return [self._measure_builtin_single(rule, rule_sql[rule.rule_uid]) for rule in rules] elapsed_ms = (time.monotonic() - started) * 1000 observed_by_uid = {str(row[0]): row[1] for row in records or []} From 790ca724845dbd69e09af74b9bec46a5a1b89f4f Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Wed, 15 Jul 2026 16:30:03 +0100 Subject: [PATCH 17/19] Fixup observed values usage --- .../airflow/providers/common/dataquality/engines/sql.py | 2 +- .../tests/unit/common/dataquality/engines/test_sql.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py index e5c9ac884b0b6..92fa6c2d2a7f9 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py @@ -124,7 +124,7 @@ def _measure_builtin_single(self, rule: DQRule, sql: str) -> Observation: return Observation( rule=rule, duration_ms=elapsed_ms, error_message="No result returned for rule", sql=sql ) - return Observation(rule=rule, observed_value=row[0], duration_ms=elapsed_ms, sql=sql) + return Observation(rule=rule, observed_value=row[1], duration_ms=elapsed_ms, sql=sql) def _measure_custom(self, rule: DQRule, table: str) -> Observation: if rule.sql is None: diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py index eebae09b192ab..1dea65ee08615 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py @@ -89,7 +89,7 @@ def test_builds_each_rules_sql_exactly_once_on_success(self, hook): def test_builds_each_rules_sql_exactly_once_on_batch_failure(self, hook): hook.get_records.side_effect = RuntimeError("connection refused") - hook.get_first.return_value = (0,) + hook.get_first.side_effect = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 0)] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) engine = SQLDQEngine(hook) @@ -100,7 +100,7 @@ def test_builds_each_rules_sql_exactly_once_on_batch_failure(self, hook): def test_batch_failure_falls_back_to_per_rule_queries(self, hook): hook.get_records.side_effect = RuntimeError("connection refused") - hook.get_first.return_value = (0,) + hook.get_first.side_effect = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 0)] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) observations = SQLDQEngine(hook).measure(ruleset, "orders") @@ -114,7 +114,10 @@ def test_batch_failure_falls_back_to_per_rule_queries(self, hook): def test_batch_failure_fallback_isolates_a_single_bad_rule(self, hook): """One rule's query failing in the per-rule fallback must not fail the other rules.""" hook.get_records.side_effect = RuntimeError("connection refused") - hook.get_first.side_effect = [(0,), RuntimeError("no such column: no_such_column")] + hook.get_first.side_effect = [ + (NULLS.rule_uid, 0), + RuntimeError("no such column: no_such_column"), + ] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) observations = SQLDQEngine(hook).measure(ruleset, "orders") From 76151f4ab297276f3319b6a6fffa76cd8017c24c Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Wed, 15 Jul 2026 18:26:46 +0100 Subject: [PATCH 18/19] Add reusable data quality execution helpers to run quality checks from any tasks --- .../common/dataquality/docs/operators.rst | 13 + .../common/dataquality/decorators/dq_check.py | 2 + .../example_dags/example_dq_in_taskflow.py | 119 ++++++++ .../providers/common/dataquality/execution.py | 236 +++++++++++++++ .../common/dataquality/operators/dq_check.py | 139 ++------- .../dataquality/decorators/test_dq_check.py | 2 +- .../dataquality/operators/test_dq_check.py | 4 +- .../unit/common/dataquality/test_execution.py | 283 ++++++++++++++++++ 8 files changed, 675 insertions(+), 123 deletions(-) create mode 100644 providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py create mode 100644 providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py create mode 100644 providers/common/dataquality/tests/unit/common/dataquality/test_execution.py diff --git a/providers/common/dataquality/docs/operators.rst b/providers/common/dataquality/docs/operators.rst index ab5eec1669b74..d12b0dd79d524 100644 --- a/providers/common/dataquality/docs/operators.rst +++ b/providers/common/dataquality/docs/operators.rst @@ -75,6 +75,19 @@ fails normally -- only persisted data quality history is unavailable. There is n per-operator override: every check in a deployment shares one results store, so history stays available across tasks and Dags without stitching together several stores. +Run quality checks inside your own tasks +------------------------------------------ + +If quality checks are one step inside a larger task, use +:func:`~airflow.providers.common.dataquality.execution.run_quality_checks` directly, then call +:func:`~airflow.providers.common.dataquality.execution.persist_quality_results` to write the same +history records that ``DQCheckOperator`` writes: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py + :language: python + :start-after: [START howto_run_dq_inside_task] + :end-before: [END howto_run_dq_inside_task] + Checking an asset -------------------- diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py index 6bd05a2e08e8f..ba4285c063fd1 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py @@ -77,6 +77,8 @@ def execute(self, context: Context) -> Any: f"got {type(ruleset).__name__}" ) self.ruleset = ruleset + elif self.ruleset is SET_DURING_EXECUTION: + raise ValueError("ruleset is required, either directly or returned by @task.dq_check") return DQCheckOperator.execute(self, context) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py new file mode 100644 index 0000000000000..e3b1f470f831a --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Dag: run data quality checks as part of a TaskFlow task. + +The task owns the surrounding Python flow -- it can persist DQ results, inspect the summary, +and decide what to return or raise -- while the rule execution still uses the same +``DbApiHook`` path as ``DQCheckOperator``. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.execution import persist_quality_results, run_quality_checks +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, get_current_context, task + +DAG_ID = "example_dq_in_taskflow" +CONN_ID = "sqlite_default" +RAW_TABLE = "dq_taskflow_raw_orders" +READY_TABLE = "dq_taskflow_ready_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +orders_ruleset = RuleSet( + name="orders_taskflow_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition=Condition(equal_to=0), + ), + DQRule( + name="amount_min_ge_zero", + check="min", + column="amount", + condition=Condition(geq_to=0), + ), + DQRule( + name="row_count_present", + check="row_count", + condition=Condition(greater_than=0), + ), + ), +) + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_tables = SQLExecuteQueryOperator( + task_id="create_tables", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {RAW_TABLE};", + f"DROP TABLE IF EXISTS {READY_TABLE};", + f"CREATE TABLE {RAW_TABLE} (order_id INTEGER, customer_id INTEGER, amount REAL);", + f"CREATE TABLE {READY_TABLE} (order_id INTEGER, customer_id INTEGER, amount REAL);", + ], + ) + + load_orders = SQLExecuteQueryOperator( + task_id="load_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {RAW_TABLE} (order_id, customer_id, amount) VALUES + (1, 101, 10.0), + (2, 102, 25.5), + (3, 103, 7.25); + """, + ) + + # [START howto_run_dq_inside_task] + @task + def validate_and_choose_source() -> str: + result = run_quality_checks( + conn_id=CONN_ID, + table=RAW_TABLE, + ruleset=orders_ruleset, + ) + summary = persist_quality_results(result, context=get_current_context()) + + if summary["failed"] or summary["errored"]: + raise ValueError(f"Order quality checks failed: {summary}") + + return RAW_TABLE + + # [END howto_run_dq_inside_task] + + publish_orders = SQLExecuteQueryOperator( + task_id="publish_orders", + conn_id=CONN_ID, + sql=f"INSERT INTO {READY_TABLE} SELECT * FROM {{{{ ti.xcom_pull(task_ids='validate_and_choose_source') }}}};", + ) + + create_tables >> load_orders >> validate_and_choose_source() >> publish_orders diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py new file mode 100644 index 0000000000000..4da476aeac491 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py @@ -0,0 +1,236 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Helpers for running data quality rules from operators or custom Python tasks.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, cast + +from airflow.providers.common.compat.sdk import BaseHook +from airflow.providers.common.dataquality.assets import DQ_RESULT_EXTRA_KEY +from airflow.providers.common.dataquality.backends import get_backend_from_config +from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.results import ( + ERROR, + FAIL, + PASS, + WARN, + DQRun, + RuleResult, + build_summary, +) +from airflow.providers.common.dataquality.rules import RuleSet, describe_rule + +if TYPE_CHECKING: + from collections.abc import Sequence + + from airflow.providers.common.dataquality.engines.sql import Observation + from airflow.providers.common.dataquality.rules.rule import Condition, Dimension + from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.sdk import Asset, Context + +log = logging.getLogger(__name__) + +RulesetArg = RuleSet | dict[str, Any] | str + + +@dataclass(frozen=True) +class DataQualityResult: + """Evaluated data quality results before Airflow task metadata is attached.""" + + ruleset: RuleSet + table: str + results: tuple[RuleResult, ...] + started_at: str + finished_at: str + + def to_dict(self) -> dict[str, Any]: + return { + "ruleset": self.ruleset.name, + "table": self.table, + "started_at": self.started_at, + "finished_at": self.finished_at, + "results": [result.to_dict() for result in self.results], + } + + +def run_quality_checks( + *, + ruleset: RulesetArg, + table: str, + conn_id: str | None = None, + hook: DbApiHook | None = None, + hook_params: dict[str, Any] | None = None, + partition_clause: str | None = None, +) -> DataQualityResult: + """ + Run data quality rules through a ``DbApiHook``-compatible connection. + + Use this from custom Python tasks when the standard + :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` + does not fit the task shape. + """ + resolved_ruleset = _resolve_ruleset(ruleset) + resolved_hook = hook if hook is not None else _get_hook(conn_id, hook_params) + started_at = datetime.now(tz=timezone.utc).isoformat() + observations = SQLDQEngine(resolved_hook).measure(resolved_ruleset, table, partition_clause) + finished_at = datetime.now(tz=timezone.utc).isoformat() + return DataQualityResult( + ruleset=resolved_ruleset, + table=table, + results=tuple(_evaluate_observation(observation) for observation in observations), + started_at=started_at, + finished_at=finished_at, + ) + + +def persist_quality_results( + result: DataQualityResult, + *, + context: Context, + outlets: Sequence[Asset] | None = None, +) -> dict[str, Any]: + """ + Persist data quality results for the current task and return the run summary. + + When ``[common.dataquality] results_path`` is not configured, this still returns + the same summary and skips writing history. When ``outlets`` are supplied, the + summary is also attached to outlet asset events for ``require_quality()``. + """ + run = _build_run_from_context( + context=context, + ruleset=result.ruleset, + table=result.table, + outlets=outlets or (), + started_at=result.started_at, + finished_at=result.finished_at, + ) + results = list(result.results) + summary = build_summary(run, results) + + backend = get_backend_from_config() + if backend is None: + log.info("No [common.dataquality] results_path configured; skipping results persistence") + else: + # Persistence is best-effort: an unreachable results store leaves a gap in + # history but must not change the outcome of the check itself. + try: + backend.write_run(run, results) + except Exception: + log.exception("Failed to persist data quality results; continuing") + + if outlets: + _attach_to_outlet_events(context, outlets, summary) + + return summary + + +def _get_hook(conn_id: str | None, hook_params: dict[str, Any] | None) -> DbApiHook: + if conn_id is None: + raise ValueError("Either conn_id or hook is required") + connection = BaseHook.get_connection(conn_id) + return connection.get_hook(hook_params=hook_params) + + +def _resolve_ruleset(ruleset: RulesetArg) -> RuleSet: + if isinstance(ruleset, RuleSet): + return ruleset + if isinstance(ruleset, dict): + return RuleSet.from_dict(ruleset) + return RuleSet.from_file(ruleset) + + +def _evaluate_observation(observation: Observation) -> RuleResult: + rule = observation.rule + condition = cast("Condition", rule.condition) + dimension = cast("Dimension", rule.dimension) + error_message = observation.error_message + if error_message is None: + try: + passed = condition.evaluate(observation.observed_value) + except (TypeError, ValueError) as e: + passed = False + error_message = f"Could not evaluate observed value {observation.observed_value!r}: {e}" + else: + passed = False + + if error_message is not None: + status = ERROR + elif passed: + status = PASS + else: + status = WARN if rule.severity == "warn" else FAIL + observed = observation.observed_value + if observed is not None and not isinstance(observed, int | float | str): + observed = str(observed) + return RuleResult( + rule_uid=rule.rule_uid, + rule_name=rule.name, + status=status, + observed_value=observed, + condition=condition.to_dict(), + dimension=dimension.value, + severity=rule.severity.value, + duration_ms=observation.duration_ms, + error_message=error_message, + description=rule.description or describe_rule(rule), + sql=observation.sql, + ) + + +def _build_run_from_context( + *, + context: Context, + ruleset: RuleSet, + table: str, + outlets: Sequence[Asset], + started_at: str, + finished_at: str, +) -> DQRun: + ti = context["ti"] + return DQRun( + dag_id=ti.dag_id, + task_id=ti.task_id, + run_id=ti.run_id, + try_number=ti.try_number, + map_index=ti.map_index if ti.map_index is not None else -1, + ruleset_name=ruleset.name, + table_ref=table, + asset_names=tuple(_get_outlet_asset_names(outlets)), + started_at=started_at, + finished_at=finished_at, + ) + + +def _get_outlet_asset_names(outlets: Sequence[Asset]) -> list[str]: + return [asset.name for asset in outlets if getattr(asset, "name", None)] + + +def _attach_to_outlet_events(context: Context, outlets: Sequence[Asset], summary: dict[str, Any]) -> None: + try: + outlet_events = context["outlet_events"] + except KeyError: + return + for outlet in outlets: + if getattr(outlet, "name", None): + try: + outlet_events[outlet].extra[DQ_RESULT_EXTRA_KEY] = summary + except Exception: + log.warning("Could not attach data quality summary to outlet event for %s", outlet) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py index 3e13ede36daba..fa2a7b5f28c1f 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py @@ -18,30 +18,26 @@ import logging from collections.abc import Sequence -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any -from airflow.providers.common.dataquality.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config -from airflow.providers.common.dataquality.backends import get_backend_from_config -from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.assets import get_asset_quality_config from airflow.providers.common.dataquality.exceptions import DQCheckFailedError +from airflow.providers.common.dataquality.execution import ( + DataQualityResult, + _resolve_ruleset, + persist_quality_results, + run_quality_checks, +) from airflow.providers.common.dataquality.results import ( ERROR, FAIL, - PASS, WARN, - DQRun, RuleResult, - build_summary, ) -from airflow.providers.common.dataquality.rules import RuleSet, describe_rule from airflow.providers.common.sql.operators.sql import BaseSQLOperator -from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION if TYPE_CHECKING: - from airflow.providers.common.dataquality.engines.sql import Observation - from airflow.providers.common.dataquality.rules.rule import Condition, Dimension - from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.providers.common.dataquality.rules import RuleSet from airflow.sdk import Asset, Context log = logging.getLogger(__name__) @@ -118,117 +114,20 @@ def __init__( self.fail_on = fail_on def execute(self, context: Context) -> dict[str, Any]: - ruleset = self._resolve_ruleset() - started_at = datetime.now(tz=timezone.utc).isoformat() - engine = self._get_engine(self.get_db_hook()) - observations = engine.measure(ruleset, self.table, self.partition_clause) - results = [self._evaluate(observation) for observation in observations] - finished_at = datetime.now(tz=timezone.utc).isoformat() - - run = self._build_run(context, ruleset, started_at, finished_at) - summary = build_summary(run, results) + result = run_quality_checks( + hook=self.get_db_hook(), + ruleset=_resolve_ruleset(self.ruleset), + table=self.table, + partition_clause=self.partition_clause, + ) + results = list(result.results) + summary = self._persist_result(result, context) self._log_results(results, summary) - self._persist(run, results) - self._attach_to_outlet_events(context, summary) self._raise_for_failures(results, summary) return summary - def _get_engine(self, hook: DbApiHook) -> SQLDQEngine: - return SQLDQEngine(hook) - - def _resolve_ruleset(self) -> RuleSet: - if self.ruleset is SET_DURING_EXECUTION: - raise ValueError("ruleset is required, either directly or returned by @task.dq_check") - if isinstance(self.ruleset, RuleSet): - return self.ruleset - if isinstance(self.ruleset, dict): - return RuleSet.from_dict(self.ruleset) - return RuleSet.from_file(self.ruleset) - - @staticmethod - def _evaluate(observation: Observation) -> RuleResult: - rule = observation.rule - condition = cast("Condition", rule.condition) - dimension = cast("Dimension", rule.dimension) - error_message = observation.error_message - if error_message is None: - try: - passed = condition.evaluate(observation.observed_value) - except (TypeError, ValueError) as e: - passed = False - error_message = f"Could not evaluate observed value {observation.observed_value!r}: {e}" - else: - passed = False - - if error_message is not None: - status = ERROR - elif passed: - status = PASS - else: - status = WARN if rule.severity == "warn" else FAIL - observed = observation.observed_value - if observed is not None and not isinstance(observed, int | float | str): - observed = str(observed) - return RuleResult( - rule_uid=rule.rule_uid, - rule_name=rule.name, - status=status, - observed_value=observed, - condition=condition.to_dict(), - dimension=dimension.value, - severity=rule.severity.value, - duration_ms=observation.duration_ms, - error_message=error_message, - description=rule.description or describe_rule(rule), - sql=observation.sql, - ) - - def _build_run(self, context: Context, ruleset: RuleSet, started_at: str, finished_at: str) -> DQRun: - ti = context["ti"] - return DQRun( - dag_id=ti.dag_id, - task_id=ti.task_id, - run_id=ti.run_id, - try_number=ti.try_number, - map_index=ti.map_index if ti.map_index is not None else -1, - ruleset_name=ruleset.name, - table_ref=self.table, - asset_names=tuple(self._outlet_asset_names()), - started_at=started_at, - finished_at=finished_at, - ) - - def _outlet_asset_names(self) -> list[str]: - names = [] - for outlet in self.outlets or []: - name = getattr(outlet, "name", None) - if name: - names.append(name) - return names - - def _persist(self, run: DQRun, results: list[RuleResult]) -> None: - backend = get_backend_from_config() - if backend is None: - self.log.info("No [common.dataquality] results_path configured; skipping results persistence") - return - # Persistence is best-effort: an unreachable results store leaves a gap in - # history but must not change the outcome of the check itself. - try: - backend.write_run(run, results) - except Exception: - self.log.exception("Failed to persist data quality results; continuing") - - def _attach_to_outlet_events(self, context: Context, summary: dict[str, Any]) -> None: - try: - outlet_events = context["outlet_events"] - except KeyError: - return - for outlet in self.outlets or []: - if getattr(outlet, "name", None): - try: - outlet_events[outlet].extra[DQ_RESULT_EXTRA_KEY] = summary - except Exception: - self.log.warning("Could not attach dq summary to outlet event for %s", outlet) + def _persist_result(self, result: DataQualityResult, context: Context) -> dict[str, Any]: + return persist_quality_results(result, context=context, outlets=list(self.outlets or ())) def _log_results(self, results: list[RuleResult], summary: dict[str, Any]) -> None: for result in results: diff --git a/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py index 59b2520034ee8..d532b0e09a18d 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py @@ -47,7 +47,7 @@ def results_backend(tmp_path): """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") with mock.patch( - "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + "airflow.providers.common.dataquality.execution.get_backend_from_config", return_value=backend, ): yield backend diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py index a0ea1af5080a4..8a8be6b84fb54 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -63,7 +63,7 @@ def results_backend(tmp_path): """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") with mock.patch( - "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + "airflow.providers.common.dataquality.execution.get_backend_from_config", return_value=backend, ): yield backend @@ -237,7 +237,7 @@ def test_backend_failure_does_not_fail_the_check(self, mock_get_db_hook): mock_get_db_hook.return_value = hook with mock.patch( - "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + "airflow.providers.common.dataquality.execution.get_backend_from_config", return_value=backend, ): summary = operator.execute(make_context()) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py b/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py new file mode 100644 index 0000000000000..4ece556fe4d99 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py @@ -0,0 +1,283 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.engines.sql import Observation +from airflow.providers.common.dataquality.execution import ( + _attach_to_outlet_events, + _build_run_from_context, + _evaluate_observation, + _get_hook, + _get_outlet_asset_names, + _resolve_ruleset, + persist_quality_results, + run_quality_checks, +) +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.sdk import Asset +from airflow.sdk.execution_time.context import OutletEventAccessors + +ORDER_ID_NOT_NULL = DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition=Condition(equal_to=0), +) +ORDER_AMOUNT_VALID = DQRule( + name="order_amount_valid", + check="min", + column="amount", + condition=Condition(geq_to=0), +) +ORDER_RULESET = RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL, ORDER_AMOUNT_VALID)) + + +def make_context(): + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "custom_quality_task" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +class TestRunQualityChecks: + def test_runs_rules_with_supplied_hook(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [ + (ORDER_ID_NOT_NULL.rule_uid, 0), + (ORDER_AMOUNT_VALID.rule_uid, 5), + ] + + result = run_quality_checks(hook=hook, table="orders", ruleset=ORDER_RULESET) + + assert result.ruleset == ORDER_RULESET + assert result.table == "orders" + assert [rule_result.status for rule_result in result.results] == ["pass", "pass"] + hook.get_records.assert_called_once() + + @mock.patch("airflow.providers.common.dataquality.execution.BaseHook.get_connection", autospec=True) + def test_runs_rules_with_conn_id(self, mock_get_connection): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, 1)] + connection = mock.Mock(spec=["get_hook"]) + connection.get_hook.return_value = hook + mock_get_connection.return_value = connection + + result = run_quality_checks( + conn_id="warehouse", + hook_params={"schema": "analytics"}, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + + assert result.results[0].status == "fail" + mock_get_connection.assert_called_once_with("warehouse") + connection.get_hook.assert_called_once_with(hook_params={"schema": "analytics"}) + + def test_returns_error_result_when_value_cannot_be_evaluated(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, "not-a-number")] + + result = run_quality_checks( + hook=hook, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + + assert result.results[0].status == "error" + assert "Could not evaluate observed value" in result.results[0].error_message + + +class TestPersistQualityResults: + def test_persists_results_for_custom_task(self, tmp_path): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [ + (ORDER_ID_NOT_NULL.rule_uid, 0), + (ORDER_AMOUNT_VALID.rule_uid, 5), + ] + result = run_quality_checks(hook=hook, table="orders", ruleset=ORDER_RULESET) + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", + return_value=backend, + ): + summary = persist_quality_results(result, context=make_context()) + + assert summary["passed"] == 2 + history = backend.read_task_rule_history( + "orders_pipeline", "custom_quality_task", ORDER_ID_NOT_NULL.rule_uid + )["items"] + assert len(history) == 1 + assert history[0]["status"] == "pass" + + def test_attaches_summary_to_outlet_asset_event(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, 0)] + result = run_quality_checks( + hook=hook, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + asset = Asset("orders") + context = make_context() + + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", return_value=None + ): + summary = persist_quality_results(result, context=context, outlets=[asset]) + + outlet_events = context["outlet_events"] + outlet_events.__getitem__.assert_called_with(asset) + outlet_events.__getitem__.return_value.extra.__setitem__.assert_called_once_with( + "airflow.dataquality.result", summary + ) + + def test_backend_failure_does_not_fail_custom_task(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, 0)] + result = run_quality_checks( + hook=hook, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + backend = mock.create_autospec(ObjectStorageResultsBackend, instance=True) + backend.write_run.side_effect = OSError("bucket unreachable") + + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", + return_value=backend, + ): + summary = persist_quality_results(result, context=make_context()) + + assert summary["passed"] == 1 + backend.write_run.assert_called_once() + + +class TestPrivateHelpers: + @pytest.mark.parametrize( + "ruleset_arg", + [ + ORDER_RULESET, + ORDER_RULESET.to_dict(), + ], + ) + def test_resolve_ruleset_from_model_or_dict(self, ruleset_arg): + assert _resolve_ruleset(ruleset_arg) == ORDER_RULESET + + def test_resolve_ruleset_from_file(self, tmp_path): + ruleset_file = tmp_path / "orders_ruleset.yaml" + ruleset_file.write_text( + """ +name: orders_quality +rules: + - name: order_id_not_null + check: null_count + column: order_id + condition: + equal_to: 0 +""", + ) + + ruleset = _resolve_ruleset(str(ruleset_file)) + + assert ruleset.name == "orders_quality" + assert ruleset.rules[0].name == "order_id_not_null" + + def test_get_hook_requires_conn_id_when_hook_is_not_supplied(self): + with pytest.raises(ValueError, match="Either conn_id or hook is required"): + _get_hook(None, None) + + @mock.patch("airflow.providers.common.dataquality.execution.BaseHook.get_connection", autospec=True) + def test_get_hook_uses_conn_id_and_hook_params(self, mock_get_connection): + hook = mock.create_autospec(DbApiHook, instance=True) + connection = mock.Mock(spec=["get_hook"]) + connection.get_hook.return_value = hook + mock_get_connection.return_value = connection + + assert _get_hook("warehouse", {"schema": "analytics"}) == hook + mock_get_connection.assert_called_once_with("warehouse") + connection.get_hook.assert_called_once_with(hook_params={"schema": "analytics"}) + + def test_evaluate_observation_records_warn_for_warn_severity_failure(self): + warn_rule = DQRule( + name="volume", + check="row_count", + condition=Condition(greater_than=100), + severity="warn", + ) + + result = _evaluate_observation(Observation(rule=warn_rule, observed_value=5, duration_ms=12.3)) + + assert result.status == "warn" + assert result.observed_value == 5 + assert result.duration_ms == 12.3 + + def test_evaluate_observation_records_error_for_query_error(self): + result = _evaluate_observation(Observation(rule=ORDER_ID_NOT_NULL, error_message="query failed")) + + assert result.status == "error" + assert result.error_message == "query failed" + + def test_build_run_from_context_uses_task_metadata_and_outlets(self): + context = make_context() + context["ti"].map_index = None + asset = Asset("orders") + + run = _build_run_from_context( + context=context, + ruleset=ORDER_RULESET, + table="orders", + outlets=[asset], + started_at="2026-07-04T12:00:00+00:00", + finished_at="2026-07-04T12:00:01+00:00", + ) + + assert run.dag_id == "orders_pipeline" + assert run.task_id == "custom_quality_task" + assert run.map_index == -1 + assert run.ruleset_name == "orders_quality" + assert run.table_ref == "orders" + assert run.asset_names == ("orders",) + assert run.started_at == "2026-07-04T12:00:00+00:00" + assert run.finished_at == "2026-07-04T12:00:01+00:00" + + def test_get_outlet_asset_names_ignores_outlets_without_names(self): + named_asset = Asset("orders") + unnamed_outlet = mock.Mock(spec=[]) + + assert _get_outlet_asset_names([named_asset, unnamed_outlet]) == ["orders"] + + def test_attach_to_outlet_events_noops_without_context_key(self): + _attach_to_outlet_events({}, [Asset("orders")], {"score": 1.0}) + + def test_attach_to_outlet_events_ignores_single_outlet_failure(self): + asset = Asset("orders") + outlet_events = mock.create_autospec(OutletEventAccessors, instance=True) + outlet_events.__getitem__.side_effect = RuntimeError("not available") + + _attach_to_outlet_events({"outlet_events": outlet_events}, [asset], {"score": 1.0}) + + outlet_events.__getitem__.assert_called_once_with(asset) From c0d1bfb87a53c4b7bee31e1cf2e9677a92fbafa5 Mon Sep 17 00:00:00 2001 From: gopidesupavan Date: Fri, 17 Jul 2026 11:13:43 +0100 Subject: [PATCH 19/19] Resolve comments from wei --- providers/common/dataquality/docs/rules.rst | 24 +-- .../dataquality/backends/object_storage.py | 77 ++++---- .../common/dataquality/engines/sql.py | 31 +-- .../example_dags/example_dq_in_taskflow.py | 6 +- .../providers/common/dataquality/execution.py | 10 +- .../providers/common/dataquality/results.py | 2 +- .../backends/test_object_storage.py | 178 +++++++++++------- .../common/dataquality/engines/test_sql.py | 28 +-- .../dataquality/operators/test_dq_check.py | 20 +- .../unit/common/dataquality/test_execution.py | 4 +- .../unit/common/dataquality/test_results.py | 2 +- 11 files changed, 218 insertions(+), 164 deletions(-) diff --git a/providers/common/dataquality/docs/rules.rst b/providers/common/dataquality/docs/rules.rst index ab59fc7e7f8ab..efcd8c0e0f136 100644 --- a/providers/common/dataquality/docs/rules.rst +++ b/providers/common/dataquality/docs/rules.rst @@ -26,26 +26,10 @@ their own. They describe *what* to check; :class:`~airflow.providers.common.data (see :doc:`operators`) is what actually runs them. Rules are usually written by hand, but they can also be proposed by an LLM -- see :doc:`agents`. -.. code-block:: python - - from airflow.providers.common.dataquality.rules import DQRule, RuleSet - - orders_ruleset = RuleSet( - name="orders_quality", - rules=( - DQRule( - name="order_id_not_null", - check="null_count", - column="order_id", - condition={"equal_to": 0}, - ), - DQRule( - name="row_count_present", - check="row_count", - condition={"greater_than": 0}, - ), - ), - ) +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py + :language: python + :start-after: [START howto_rules_dq_ruleset] + :end-before: [END howto_rules_dq_ruleset] ``DQRule`` fields ------------------ diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py index c640d24abe61c..34003bdda8a06 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py @@ -37,7 +37,7 @@ import json import logging from datetime import datetime, timezone -from typing import Any +from typing import Any, TypedDict from airflow.providers.common.dataquality.results import DQRun, RuleResult, build_summary from airflow.sdk import ObjectStoragePath @@ -45,24 +45,31 @@ log = logging.getLogger(__name__) +class DQPageResult(TypedDict): + """Paginated DQ read result.""" + + items: list[dict[str, Any]] + next_cursor: str | None + + class ObjectStorageResultsBackend: """Persist DQ results as JSON files via ``ObjectStoragePath``.""" - def __init__(self, results_path: str, conn_id: str | None = None) -> None: + def __init__(self, *, results_path: str, conn_id: str | None = None) -> None: self.root = ObjectStoragePath(results_path, conn_id=conn_id) - def write_run(self, run: DQRun, results: list[RuleResult]) -> None: + def write_run(self, *, run: DQRun, results: list[RuleResult]) -> None: timestamp = run.started_at or datetime.now(tz=timezone.utc).isoformat() compact_ts = self._get_safe_key(timestamp) - payload = self._build_run_payload(run, results) + payload = self._build_run_payload(run=run, results=results) - self._write_run_file(run, timestamp[:10], compact_ts, payload) - self._write_task_instance_index(run, payload) - self._write_rule_indexes(run, results, timestamp) + self._write_run_file(run=run, date_part=timestamp[:10], compact_ts=compact_ts, payload=payload) + self._write_task_instance_index(run=run, payload=payload) + self._write_rule_indexes(run=run, results=results, timestamp=timestamp) def read_task_rule_history( - self, dag_id: str, task_id: str, rule_uid: str, limit: int = 100, before: str | None = None - ) -> dict[str, Any]: + self, *, dag_id: str, task_id: str, rule_uid: str, limit: int = 100, before: str | None = None + ) -> DQPageResult: """Return recent results for one rule produced by one task, newest first.""" rule_dir = ( self.root @@ -72,11 +79,11 @@ def read_task_rule_history( / f"task_id={task_id}" / f"rule_uid={rule_uid}" ) - return self._read_rule_history_dir(rule_dir, limit, before) + return self._read_rule_history_dir(rule_dir=rule_dir, limit=limit, before=before) def read_task_runs( - self, dag_id: str, task_id: str, limit: int = 50, before: str | None = None - ) -> dict[str, Any]: + self, *, dag_id: str, task_id: str, limit: int = 50, before: str | None = None + ) -> DQPageResult: """ Return recent data quality runs for one task, newest first. @@ -103,11 +110,9 @@ def read_task_runs( for path in sorted(date_dir.iterdir(), key=lambda p: p.name, reverse=True): if not path.name.endswith(".json"): continue - payload = self._read_json(path) - if payload is None: + if (payload := self._read_json(path)) is None: continue - cursor = self._get_run_payload_cursor(payload) - if before is not None and cursor >= before: + if before is not None and self._get_run_payload_cursor(payload) >= before: continue runs.append(payload) if len(runs) > limit: @@ -121,7 +126,7 @@ def read_task_runs( return {"items": page, "next_cursor": next_cursor} def read_by_task_instance( - self, dag_id: str, task_id: str, run_id: str, map_index: int = -1 + self, *, dag_id: str, task_id: str, run_id: str, map_index: int = -1 ) -> dict[str, Any]: """Read the latest run for one task instance as ``{"run": ..., "results": ..., "summary": ...}``.""" path = ( @@ -134,7 +139,9 @@ def read_by_task_instance( ) return self._read_json_or_raise(path) - def _write_run_file(self, run: DQRun, date_part: str, compact_ts: str, payload: dict[str, Any]) -> None: + def _write_run_file( + self, *, run: DQRun, date_part: str, compact_ts: str, payload: dict[str, Any] + ) -> None: run_dir = ( self.root / "runs" @@ -146,39 +153,41 @@ def _write_run_file(self, run: DQRun, date_part: str, compact_ts: str, payload: run_dir.mkdir(parents=True, exist_ok=True) (run_dir / f"{compact_ts}__{run.run_uid}.json").write_text(json.dumps(payload, default=str)) - def _write_task_instance_index(self, run: DQRun, payload: dict[str, Any]) -> None: + def _write_task_instance_index(self, *, run: DQRun, payload: dict[str, Any]) -> None: ti_dir = self.root / "runs" / "by_task_instance" / f"dag_id={run.dag_id}" / f"task_id={run.task_id}" ti_dir.mkdir(parents=True, exist_ok=True) (ti_dir / f"{self._get_safe_key(run.run_id)}__{run.map_index}.json").write_text( json.dumps(payload, default=str) ) - def _write_rule_indexes(self, run: DQRun, results: list[RuleResult], timestamp: str) -> None: + def _write_rule_indexes(self, *, run: DQRun, results: list[RuleResult], timestamp: str) -> None: run_context = self._build_run_context(run) compact_ts = self._get_safe_key(timestamp) for result in results: payload = {"run": run_context, "result": result.to_dict()} self._write_rule_index( - self.root - / "rules" - / "by_task_rule" - / f"dag_id={run.dag_id}" - / f"task_id={run.task_id}" - / f"rule_uid={result.rule_uid}", - compact_ts, - run.run_uid, - payload, + rule_dir=( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"rule_uid={result.rule_uid}" + ), + compact_ts=compact_ts, + run_uid=run.run_uid, + payload=payload, ) def _write_rule_index( - self, rule_dir: ObjectStoragePath, compact_ts: str, run_uid: str, payload: dict[str, Any] + self, *, rule_dir: ObjectStoragePath, compact_ts: str, run_uid: str, payload: dict[str, Any] ) -> None: rule_dir.mkdir(parents=True, exist_ok=True) (rule_dir / f"{compact_ts}__{run_uid}.json").write_text(json.dumps(payload, default=str)) def _read_rule_history_dir( - self, rule_dir: ObjectStoragePath, limit: int, before: str | None = None - ) -> dict[str, Any]: + self, *, rule_dir: ObjectStoragePath, limit: int, before: str | None = None + ) -> DQPageResult: """ Read rule-result records newest-first, as ``{"items": [...], "next_cursor": ...}``. @@ -211,12 +220,12 @@ def _get_safe_key(value: str) -> str: return value.replace("/", "_").replace(":", "_").replace("+", "_") @staticmethod - def _build_run_payload(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + def _build_run_payload(*, run: DQRun, results: list[RuleResult]) -> dict[str, Any]: result_records = [result.to_dict() for result in results] return { "run": run.to_dict(), "results": result_records, - "summary": build_summary(run, results), + "summary": build_summary(run=run, results=results), } @staticmethod diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py index 92fa6c2d2a7f9..abfb468c2ff17 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py @@ -57,21 +57,27 @@ class SQLDQEngine: def __init__(self, hook: DbApiHook) -> None: self.hook = hook - def measure(self, ruleset: RuleSet, table: str, partition_clause: str | None = None) -> list[Observation]: + def measure( + self, *, ruleset: RuleSet, table: str, partition_clause: str | None = None + ) -> list[Observation]: builtin_rules = [rule for rule in ruleset.rules if rule.check != CUSTOM_SQL_CHECK] custom_rules = [rule for rule in ruleset.rules if rule.check == CUSTOM_SQL_CHECK] observations = [] if builtin_rules: - observations.extend(self._measure_builtin(builtin_rules, table, partition_clause)) + observations.extend( + self._measure_builtin(rules=builtin_rules, table=table, partition_clause=partition_clause) + ) for rule in custom_rules: - observations.append(self._measure_custom(rule, table)) + observations.append(self._measure_custom(rule=rule, table=table)) return observations - def build_batch_sql(self, rules: list[DQRule], table: str, partition_clause: str | None) -> str: - return " UNION ALL ".join(self.build_rule_sql(rule, table, partition_clause) for rule in rules) + def build_batch_sql(self, *, rules: list[DQRule], table: str, partition_clause: str | None) -> str: + return " UNION ALL ".join( + self.build_rule_sql(rule=rule, table=table, partition_clause=partition_clause) for rule in rules + ) - def build_rule_sql(self, rule: DQRule, table: str, partition_clause: str | None = None) -> str: + def build_rule_sql(self, *, rule: DQRule, table: str, partition_clause: str | None = None) -> str: """Build the SQL used to measure one built-in rule.""" expression = CHECK_SPECS[rule.check].expression.format(column=rule.column) predicates = [p for p in (partition_clause, rule.partition_clause) if p] @@ -81,11 +87,14 @@ def build_rule_sql(self, rule: DQRule, table: str, partition_clause: str | None ) def _measure_builtin( - self, rules: list[DQRule], table: str, partition_clause: str | None + self, *, rules: list[DQRule], table: str, partition_clause: str | None ) -> list[Observation]: # Built once per rule and reused below, instead of re-deriving each rule's SQL for the # batch, for the returned Observation, and again in the per-rule fallback. - rule_sql = {rule.rule_uid: self.build_rule_sql(rule, table, partition_clause) for rule in rules} + rule_sql = { + rule.rule_uid: self.build_rule_sql(rule=rule, table=table, partition_clause=partition_clause) + for rule in rules + } sql = " UNION ALL ".join(rule_sql.values()) log.info("Running %d built-in checks against %s", len(rules), table) started = time.monotonic() @@ -97,7 +106,7 @@ def _measure_builtin( table, ) - return [self._measure_builtin_single(rule, rule_sql[rule.rule_uid]) for rule in rules] + return [self._measure_builtin_single(rule=rule, sql=rule_sql[rule.rule_uid]) for rule in rules] elapsed_ms = (time.monotonic() - started) * 1000 observed_by_uid = {str(row[0]): row[1] for row in records or []} return [ @@ -111,7 +120,7 @@ def _measure_builtin( for rule in rules ] - def _measure_builtin_single(self, rule: DQRule, sql: str) -> Observation: + def _measure_builtin_single(self, *, rule: DQRule, sql: str) -> Observation: started = time.monotonic() try: row = self.hook.get_first(sql) @@ -126,7 +135,7 @@ def _measure_builtin_single(self, rule: DQRule, sql: str) -> Observation: ) return Observation(rule=rule, observed_value=row[1], duration_ms=elapsed_ms, sql=sql) - def _measure_custom(self, rule: DQRule, table: str) -> Observation: + def _measure_custom(self, *, rule: DQRule, table: str) -> Observation: if rule.sql is None: raise ValueError(f"Rule {rule.name!r} has no SQL to execute") sql = rule.sql.replace("{table}", table) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py index e3b1f470f831a..dcab9cfebedf8 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py @@ -39,8 +39,7 @@ READY_TABLE = "dq_taskflow_ready_orders" RESULTS_PATH = Path("/tmp/airflow_dq_example/results") -os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") - +# [START howto_rules_dq_ruleset] orders_ruleset = RuleSet( name="orders_taskflow_quality", rules=( @@ -63,6 +62,9 @@ ), ), ) +# [END howto_rules_dq_ruleset] + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") with DAG( dag_id=DAG_ID, diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py index 4da476aeac491..4b4b01d7613b2 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py @@ -90,7 +90,11 @@ def run_quality_checks( resolved_ruleset = _resolve_ruleset(ruleset) resolved_hook = hook if hook is not None else _get_hook(conn_id, hook_params) started_at = datetime.now(tz=timezone.utc).isoformat() - observations = SQLDQEngine(resolved_hook).measure(resolved_ruleset, table, partition_clause) + observations = SQLDQEngine(resolved_hook).measure( + ruleset=resolved_ruleset, + table=table, + partition_clause=partition_clause, + ) finished_at = datetime.now(tz=timezone.utc).isoformat() return DataQualityResult( ruleset=resolved_ruleset, @@ -123,7 +127,7 @@ def persist_quality_results( finished_at=result.finished_at, ) results = list(result.results) - summary = build_summary(run, results) + summary = build_summary(run=run, results=results) backend = get_backend_from_config() if backend is None: @@ -132,7 +136,7 @@ def persist_quality_results( # Persistence is best-effort: an unreachable results store leaves a gap in # history but must not change the outcome of the check itself. try: - backend.write_run(run, results) + backend.write_run(run=run, results=results) except Exception: log.exception("Failed to persist data quality results; continuing") diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py index 5fbcf04573cc7..256b147e04fac 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py @@ -100,7 +100,7 @@ def compute_score(results: list[RuleResult]) -> float | None: return round(1.0 - penalty / len(results), 4) -def build_summary(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: +def build_summary(*, run: DQRun, results: list[RuleResult]) -> dict[str, Any]: """Compact run summary attached to XCom and outlet asset events.""" return { "run_uid": run.run_uid, diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py index c05e7ec221ce3..4e67f45691893 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py @@ -62,16 +62,18 @@ def backend(self, tmp_path): return ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") def test_write_run_creates_run_file_with_run_and_results(self, backend): - backend.write_run(make_run(), [make_result()]) + backend.write_run(run=make_run(), results=[make_result()]) - record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + record = backend.read_by_task_instance( + dag_id="orders_pipeline", task_id="dq", run_id="scheduled__2026-07-04" + ) assert record["run"]["run_uid"] == "abc123" assert record["run"]["dag_id"] == "orders_pipeline" assert record["results"][0]["rule_uid"] == "rule-1" assert record["results"][0]["status"] == "pass" def test_write_run_stores_keyed_json_payload(self, backend): - backend.write_run(make_run(), [make_result()]) + backend.write_run(run=make_run(), results=[make_result()]) path = ( backend.root @@ -89,7 +91,7 @@ def test_write_run_stores_keyed_json_payload(self, backend): assert payload["summary"]["passed"] == 1 def test_write_run_stores_task_rule_index(self, backend): - backend.write_run(make_run(), [make_result()]) + backend.write_run(run=make_run(), results=[make_result()]) path = ( backend.root @@ -107,7 +109,7 @@ def test_write_run_stores_task_rule_index(self, backend): assert payload["result"]["rule_uid"] == "rule-1" def test_write_run_does_not_store_global_rule_index(self, backend): - backend.write_run(make_run(), [make_result()]) + backend.write_run(run=make_run(), results=[make_result()]) path = backend.root / "rules" / "by_rule" / "rule_uid=rule-1" @@ -115,24 +117,26 @@ def test_write_run_does_not_store_global_rule_index(self, backend): def test_rule_history_is_newest_first(self, backend): backend.write_run( - make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), - [make_result(status="pass")], + run=make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + results=[make_result(status="pass")], ) backend.write_run( - make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), - [make_result(status="fail")], + run=make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + results=[make_result(status="fail")], ) - result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1") + result = backend.read_task_rule_history(dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1") assert [record["status"] for record in result["items"]] == ["fail", "pass"] assert result["items"][0]["run"]["run_uid"] == "run2" assert result["next_cursor"] is None def test_rule_history_includes_task_instance_route_context(self, backend): - backend.write_run(make_run(), [make_result()]) + backend.write_run(run=make_run(), results=[make_result()]) - history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + history = backend.read_task_rule_history(dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1")[ + "items" + ] assert history[0]["run"]["dag_id"] == "orders_pipeline" assert history[0]["run"]["task_id"] == "dq" @@ -142,11 +146,13 @@ def test_rule_history_includes_task_instance_route_context(self, backend): def test_rule_history_respects_limit(self, backend): for day in range(1, 4): backend.write_run( - make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], ) - result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + result = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) assert len(result["items"]) == 2 assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" @@ -154,13 +160,19 @@ def test_rule_history_respects_limit(self, backend): def test_rule_history_before_cursor_pages_further_back(self, backend): for day in range(1, 4): backend.write_run( - make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], ) - first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + first_page = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) second_page = backend.read_task_rule_history( - "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + dag_id="orders_pipeline", + task_id="dq", + rule_uid="rule-1", + limit=2, + before=first_page["next_cursor"], ) assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] @@ -168,16 +180,18 @@ def test_rule_history_before_cursor_pages_further_back(self, backend): assert second_page["next_cursor"] is None def test_concurrent_style_writes_do_not_collide(self, backend): - backend.write_run(make_run(run_uid="run1"), [make_result()]) - backend.write_run(make_run(run_uid="run2"), [make_result()]) + backend.write_run(run=make_run(run_uid="run1"), results=[make_result()]) + backend.write_run(run=make_run(run_uid="run2"), results=[make_result()]) - runs = backend.read_task_runs("orders_pipeline", "dq")["items"] + runs = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq")["items"] assert {run["run"]["run_uid"] for run in runs} == {"run1", "run2"} def test_read_by_task_instance_matches_run(self, backend): - backend.write_run(make_run(), [make_result()]) + backend.write_run(run=make_run(), results=[make_result()]) - record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + record = backend.read_by_task_instance( + dag_id="orders_pipeline", task_id="dq", run_id="scheduled__2026-07-04" + ) assert record["run"]["run_uid"] == "abc123" assert record["results"][0]["rule_uid"] == "rule-1" @@ -185,15 +199,17 @@ def test_read_by_task_instance_matches_run(self, backend): def test_read_by_task_instance_unknown_raises(self, backend): with pytest.raises(FileNotFoundError): - backend.read_by_task_instance("orders_pipeline", "dq", "no_such_run") + backend.read_by_task_instance(dag_id="orders_pipeline", task_id="dq", run_id="no_such_run") def test_read_by_task_instance_last_write_wins_across_retries(self, backend): first = make_run(run_uid="try1") - backend.write_run(first, [make_result(status="fail")]) + backend.write_run(run=first, results=[make_result(status="fail")]) second = make_run(run_uid="try2") - backend.write_run(second, [make_result(status="pass")]) + backend.write_run(run=second, results=[make_result(status="pass")]) - record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + record = backend.read_by_task_instance( + dag_id="orders_pipeline", task_id="dq", run_id="scheduled__2026-07-04" + ) assert record["run"]["run_uid"] == "try2" assert record["results"][0]["status"] == "pass" @@ -202,23 +218,23 @@ def test_read_by_task_instance_last_write_wins_across_retries(self, backend): def test_read_by_task_instance_sanitizes_run_id(self, backend): run_id = "manual__2026-07-04T06:00:00+00:00" run = DQRun(dag_id="orders_pipeline", task_id="dq", run_id=run_id, run_uid="abc123") - backend.write_run(run, [make_result()]) + backend.write_run(run=run, results=[make_result()]) - record = backend.read_by_task_instance("orders_pipeline", "dq", run_id) + record = backend.read_by_task_instance(dag_id="orders_pipeline", task_id="dq", run_id=run_id) assert record["run"]["run_id"] == run_id def test_read_task_runs_returns_newest_first_with_summaries(self, backend): backend.write_run( - make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), - [make_result(status="pass")], + run=make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + results=[make_result(status="pass")], ) backend.write_run( - make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), - [make_result(status="fail")], + run=make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + results=[make_result(status="fail")], ) - result = backend.read_task_runs("orders_pipeline", "dq") + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq") runs = result["items"] assert [record["run"]["run_uid"] for record in runs] == ["run2", "run1"] @@ -229,11 +245,11 @@ def test_read_task_runs_returns_newest_first_with_summaries(self, backend): def test_read_task_runs_respects_limit(self, backend): for day in range(1, 4): backend.write_run( - make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], ) - result = backend.read_task_runs("orders_pipeline", "dq", limit=2) + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) assert len(result["items"]) == 2 assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" @@ -241,13 +257,16 @@ def test_read_task_runs_respects_limit(self, backend): def test_read_task_runs_before_cursor_pages_further_back(self, backend): for day in range(1, 4): backend.write_run( - make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], ) - first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + first_page = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) second_page = backend.read_task_runs( - "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + dag_id="orders_pipeline", + task_id="dq", + limit=2, + before=first_page["next_cursor"], ) assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] @@ -257,13 +276,16 @@ def test_read_task_runs_before_cursor_pages_further_back(self, backend): def test_read_task_runs_cursor_keeps_same_timestamp_records(self, backend): for run_uid in ("run1", "run2", "run3"): backend.write_run( - make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + results=[make_result()], ) - first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + first_page = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) second_page = backend.read_task_runs( - "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + dag_id="orders_pipeline", + task_id="dq", + limit=2, + before=first_page["next_cursor"], ) assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] @@ -273,8 +295,8 @@ def test_read_task_runs_cursor_keeps_same_timestamp_records(self, backend): def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monkeypatch): for day in range(1, 5): backend.write_run( - make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], ) read_paths: list[Any] = [] @@ -283,7 +305,7 @@ def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monk backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] ) - result = backend.read_task_runs("orders_pipeline", "dq", limit=1) + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=1) assert [record["run"]["run_uid"] for record in result["items"]] == ["run4"] assert result["next_cursor"] == "2026-07-04T06:00:00+00:00|run4" @@ -294,8 +316,8 @@ def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monk def test_read_task_runs_stops_inside_date_partition_once_limit_is_reached(self, backend, monkeypatch): for index in range(1, 5): backend.write_run( - make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), - [make_result()], + run=make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), + results=[make_result()], ) read_paths: list[Any] = [] @@ -304,7 +326,7 @@ def test_read_task_runs_stops_inside_date_partition_once_limit_is_reached(self, backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] ) - result = backend.read_task_runs("orders_pipeline", "dq", limit=1) + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=1) assert [r["run"]["run_uid"] for r in result["items"]] == ["run4"] assert result["next_cursor"] == "2026-07-04T06:00:04+00:00|run4" @@ -318,29 +340,31 @@ def test_read_task_runs_within_partition_returns_newest_first_regardless_of_writ """ for index in (3, 1, 4, 2): backend.write_run( - make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), - [make_result()], + run=make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), + results=[make_result()], ) - result = backend.read_task_runs("orders_pipeline", "dq", limit=2) + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) assert [r["run"]["run_uid"] for r in result["items"]] == ["run4", "run3"] assert result["next_cursor"] == "2026-07-04T06:00:03+00:00|run3" def test_read_task_rule_history_filters_to_task(self, backend): - backend.write_run(make_run(run_uid="run1"), [make_result()]) + backend.write_run(run=make_run(run_uid="run1"), results=[make_result()]) backend.write_run( - DQRun( + run=DQRun( dag_id="other_pipeline", task_id="dq", run_id="scheduled__2026-07-04", run_uid="run2", started_at="2026-07-04T07:00:00+00:00", ), - [make_result()], + results=[make_result()], ) - history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + history = backend.read_task_rule_history(dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1")[ + "items" + ] assert len(history) == 1 assert history[0]["run"]["dag_id"] == "orders_pipeline" @@ -348,11 +372,13 @@ def test_read_task_rule_history_filters_to_task(self, backend): def test_read_task_rule_history_respects_limit(self, backend): for day in range(1, 4): backend.write_run( - make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], ) - result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + result = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) assert [record["run"]["run_uid"] for record in result["items"]] == ["run3", "run2"] assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" @@ -360,13 +386,19 @@ def test_read_task_rule_history_respects_limit(self, backend): def test_read_task_rule_history_before_cursor_pages_further_back(self, backend): for day in range(1, 4): backend.write_run( - make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], ) - first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + first_page = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) second_page = backend.read_task_rule_history( - "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + dag_id="orders_pipeline", + task_id="dq", + rule_uid="rule-1", + limit=2, + before=first_page["next_cursor"], ) assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] @@ -375,13 +407,19 @@ def test_read_task_rule_history_before_cursor_pages_further_back(self, backend): def test_read_task_rule_history_cursor_keeps_same_timestamp_records(self, backend): for run_uid in ("run1", "run2", "run3"): backend.write_run( - make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), - [make_result()], + run=make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + results=[make_result()], ) - first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + first_page = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) second_page = backend.read_task_rule_history( - "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + dag_id="orders_pipeline", + task_id="dq", + rule_uid="rule-1", + limit=2, + before=first_page["next_cursor"], ) assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py index 1dea65ee08615..995444c95ba19 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py @@ -43,7 +43,7 @@ def hook(): class TestBuildBatchSql: def test_batch_sql_unions_one_select_per_rule(self, hook): engine = SQLDQEngine(hook) - sql = engine.build_batch_sql([NULLS, VOLUME], "orders", None) + sql = engine.build_batch_sql(rules=[NULLS, VOLUME], table="orders", partition_clause=None) assert sql.count("UNION ALL") == 1 assert f"'{NULLS.rule_uid}' AS rule_uid" in sql assert "SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END)" in sql @@ -58,7 +58,9 @@ def test_partition_clauses_are_anded(self, hook): condition=Condition(equal_to=0), partition_clause="region = 'EU'", ) - sql = SQLDQEngine(hook).build_batch_sql([rule], "orders", "ds = '2026-07-04'") + sql = SQLDQEngine(hook).build_batch_sql( + rules=[rule], table="orders", partition_clause="ds = '2026-07-04'" + ) assert "WHERE ds = '2026-07-04' AND region = 'EU'" in sql @@ -67,14 +69,14 @@ def test_builtin_rules_measured_from_single_query(self, hook): hook.get_records.return_value = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 42)] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) - observations = SQLDQEngine(hook).measure(ruleset, "orders") + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") hook.get_records.assert_called_once() by_name = {obs.rule.name: obs for obs in observations} assert by_name["nulls"].observed_value == 0 assert by_name["volume"].observed_value == 42 - assert by_name["nulls"].sql == SQLDQEngine(hook).build_rule_sql(NULLS, "orders") - assert by_name["volume"].sql == SQLDQEngine(hook).build_rule_sql(VOLUME, "orders") + assert by_name["nulls"].sql == SQLDQEngine(hook).build_rule_sql(rule=NULLS, table="orders") + assert by_name["volume"].sql == SQLDQEngine(hook).build_rule_sql(rule=VOLUME, table="orders") assert all(obs.error_message is None for obs in observations) def test_builds_each_rules_sql_exactly_once_on_success(self, hook): @@ -83,7 +85,7 @@ def test_builds_each_rules_sql_exactly_once_on_success(self, hook): engine = SQLDQEngine(hook) with mock.patch.object(engine, "build_rule_sql", wraps=engine.build_rule_sql) as build_rule_sql: - engine.measure(ruleset, "orders") + engine.measure(ruleset=ruleset, table="orders") assert build_rule_sql.call_count == 2 @@ -94,7 +96,7 @@ def test_builds_each_rules_sql_exactly_once_on_batch_failure(self, hook): engine = SQLDQEngine(hook) with mock.patch.object(engine, "build_rule_sql", wraps=engine.build_rule_sql) as build_rule_sql: - engine.measure(ruleset, "orders") + engine.measure(ruleset=ruleset, table="orders") assert build_rule_sql.call_count == 2 @@ -103,7 +105,7 @@ def test_batch_failure_falls_back_to_per_rule_queries(self, hook): hook.get_first.side_effect = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 0)] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) - observations = SQLDQEngine(hook).measure(ruleset, "orders") + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") assert hook.get_first.call_count == 2 assert len(observations) == 2 @@ -120,7 +122,7 @@ def test_batch_failure_fallback_isolates_a_single_bad_rule(self, hook): ] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) - observations = SQLDQEngine(hook).measure(ruleset, "orders") + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") by_name = {obs.rule.name: obs for obs in observations} assert by_name["nulls"].observed_value == 0 @@ -133,7 +135,7 @@ def test_per_rule_fallback_query_with_no_rows_is_an_error(self, hook): hook.get_first.return_value = None ruleset = RuleSet(name="s", rules=(NULLS,)) - observations = SQLDQEngine(hook).measure(ruleset, "orders") + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") assert observations[0].error_message == "No result returned for rule" assert observations[0].observed_value is None @@ -142,7 +144,7 @@ def test_missing_rule_in_result_is_an_error(self, hook): hook.get_records.return_value = [(NULLS.rule_uid, 0)] ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) - observations = SQLDQEngine(hook).measure(ruleset, "orders") + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") by_name = {obs.rule.name: obs for obs in observations} assert by_name["nulls"].error_message is None @@ -152,7 +154,7 @@ def test_custom_sql_rule_runs_individually_with_table_substituted(self, hook): hook.get_first.return_value = (3,) ruleset = RuleSet(name="s", rules=(CUSTOM,)) - observations = SQLDQEngine(hook).measure(ruleset, "orders") + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") hook.get_first.assert_called_once_with("SELECT COUNT(*) FROM orders WHERE amount < 0") hook.get_records.assert_not_called() @@ -163,6 +165,6 @@ def test_custom_sql_no_rows_is_an_error(self, hook): hook.get_first.return_value = None ruleset = RuleSet(name="s", rules=(CUSTOM,)) - observations = SQLDQEngine(hook).measure(ruleset, "orders") + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") assert observations[0].error_message == "Query returned no rows" diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py index 8a8be6b84fb54..1eef94d7319cd 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -131,7 +131,9 @@ def test_all_pass_returns_summary_and_persists(self, mock_get_db_hook, results_b assert summary["passed"] == 2 assert summary["failed"] == 0 assert summary["score"] == 1.0 - history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + history = results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=NULLS.rule_uid + )["items"] assert len(history) == 1 assert history[0]["status"] == "pass" @@ -145,9 +147,9 @@ def test_error_severity_failure_fails_task(self, mock_get_db_hook, results_backe # The failing result is persisted even though the task fails. assert ( - results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"][0][ - "status" - ] + results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=NULLS.rule_uid + )["items"][0]["status"] == "fail" ) @@ -179,9 +181,9 @@ def test_rule_description_is_persisted(self, mock_get_db_hook, results_backend): operator.execute(make_context()) - history = results_backend.read_task_rule_history("orders_pipeline", "dq", described_rule.rule_uid)[ - "items" - ] + history = results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=described_rule.rule_uid + )["items"] assert history[0]["description"] == "Order IDs must always be present." @mock.patch.object(DQCheckOperator, "get_db_hook") @@ -225,7 +227,9 @@ def test_non_numeric_observed_value_is_persisted_as_error(self, mock_get_db_hook with pytest.raises(DQCheckFailedError, match="errored"): operator.execute(make_context()) - history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + history = results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=NULLS.rule_uid + )["items"] assert history[0]["status"] == "error" assert "Could not evaluate observed value" in history[0]["error_message"] diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py b/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py index 4ece556fe4d99..10bc6921f928b 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py @@ -128,7 +128,9 @@ def test_persists_results_for_custom_task(self, tmp_path): assert summary["passed"] == 2 history = backend.read_task_rule_history( - "orders_pipeline", "custom_quality_task", ORDER_ID_NOT_NULL.rule_uid + dag_id="orders_pipeline", + task_id="custom_quality_task", + rule_uid=ORDER_ID_NOT_NULL.rule_uid, )["items"] assert len(history) == 1 assert history[0]["status"] == "pass" diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_results.py b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py index 37147de4416cb..3490a55af087d 100644 --- a/providers/common/dataquality/tests/unit/common/dataquality/test_results.py +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py @@ -52,7 +52,7 @@ def test_summary_counts_and_rule_names(self): make_result("error", "d"), ] - summary = build_summary(run, results) + summary = build_summary(run=run, results=results) assert summary["passed"] == 1 assert summary["warned"] == 1