From d2296699c8bd7b69e151392d7e22253182008469 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Sun, 18 Aug 2024 18:36:18 +0300 Subject: [PATCH 01/48] move bash operator to standard provider --- .../airflow_providers_bug_report.yml | 1 + .pre-commit-config.yaml | 5 +- airflow/providers/core/CHANGELOG.rst | 38 +++++++ airflow/providers/core/__init__.py | 17 +++ airflow/providers/core/provider.yaml | 43 ++++++++ airflow/providers/core/time/__init__.py | 16 +++ .../providers/core/time/operators/__init__.py | 16 +++ .../providers/core/time/sensors/__init__.py | 16 +++ .../providers/core/time/sensors/weekday.py | 104 ++++++++++++++++++ ...ement_generate-issue-content-providers.txt | 2 +- ...agement_prepare-provider-documentation.txt | 2 +- ...e-management_prepare-provider-packages.txt | 2 +- .../changelog.rst | 18 +++ .../apache-airflow-providers-core/commits.rst | 18 +++ docs/apache-airflow-providers-core/index.rst | 97 ++++++++++++++++ .../installing-providers-from-sources.rst | 18 +++ .../security.rst | 18 +++ generated/provider_dependencies.json | 10 ++ tests/providers/core/__init__.py | 16 +++ tests/providers/core/time/__init__.py | 16 +++ .../providers/core/time/operators/__init__.py | 16 +++ tests/providers/core/time/sensors/__init__.py | 16 +++ .../sensors/test_external_task_sensor.py | 0 tests/system/providers/core/__init__.py | 16 +++ tests/system/providers/core/time/__init__.py | 16 +++ .../providers/core/time/operators/__init__.py | 16 +++ .../example_branch_datetime_operator.py | 0 .../example_branch_day_of_week_operator.py | 0 .../providers/core/time/sensors/__init__.py | 16 +++ .../core/time/sensors}/example_sensors.py | 0 .../example_time_delta_sensor_async.py | 0 31 files changed, 565 insertions(+), 4 deletions(-) create mode 100644 airflow/providers/core/CHANGELOG.rst create mode 100644 airflow/providers/core/__init__.py create mode 100644 airflow/providers/core/provider.yaml create mode 100644 airflow/providers/core/time/__init__.py create mode 100644 airflow/providers/core/time/operators/__init__.py create mode 100644 airflow/providers/core/time/sensors/__init__.py create mode 100644 airflow/providers/core/time/sensors/weekday.py create mode 100644 docs/apache-airflow-providers-core/changelog.rst create mode 100644 docs/apache-airflow-providers-core/commits.rst create mode 100644 docs/apache-airflow-providers-core/index.rst create mode 100644 docs/apache-airflow-providers-core/installing-providers-from-sources.rst create mode 100644 docs/apache-airflow-providers-core/security.rst create mode 100644 tests/providers/core/__init__.py create mode 100644 tests/providers/core/time/__init__.py create mode 100644 tests/providers/core/time/operators/__init__.py create mode 100644 tests/providers/core/time/sensors/__init__.py rename tests/{ => providers/core/time}/sensors/test_external_task_sensor.py (100%) create mode 100644 tests/system/providers/core/__init__.py create mode 100644 tests/system/providers/core/time/__init__.py create mode 100644 tests/system/providers/core/time/operators/__init__.py rename {airflow/example_dags => tests/system/providers/core/time/operators}/example_branch_datetime_operator.py (100%) rename {airflow/example_dags => tests/system/providers/core/time/operators}/example_branch_day_of_week_operator.py (100%) create mode 100644 tests/system/providers/core/time/sensors/__init__.py rename {airflow/example_dags => tests/system/providers/core/time/sensors}/example_sensors.py (100%) rename {airflow/example_dags => tests/system/providers/core/time/sensors}/example_time_delta_sensor_async.py (100%) diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index 03a19e8c7753e..c5051297140ef 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -52,6 +52,7 @@ body: - common-compat - common-io - common-sql + - core - databricks - datadog - dbt-cloud diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a568c79c265aa..9f44da6afceb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -953,7 +953,10 @@ repos: entry: ./scripts/ci/pre_commit/check_system_tests.py language: python files: ^tests/system/.*/example_[^/]*\.py$ - exclude: ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ + exclude: > + (?x) + ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ | + ^tests/system/providers/core/time/operators/example_branch_datetime_operator.py pass_filenames: true additional_dependencies: ['rich>=12.4.4'] - id: generate-pypi-readme diff --git a/airflow/providers/core/CHANGELOG.rst b/airflow/providers/core/CHANGELOG.rst new file mode 100644 index 0000000000000..85db68915f4f2 --- /dev/null +++ b/airflow/providers/core/CHANGELOG.rst @@ -0,0 +1,38 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. http://www.apache.org/licenses/LICENSE-2.0 + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. +``apache-airflow-providers-core`` + + +Changelog +--------- + +1.0.0 +..... + +Initial version of the provider. diff --git a/airflow/providers/core/__init__.py b/airflow/providers/core/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/airflow/providers/core/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/core/provider.yaml b/airflow/providers/core/provider.yaml new file mode 100644 index 0000000000000..a535f4a75652c --- /dev/null +++ b/airflow/providers/core/provider.yaml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-core +name: Core +description: | + Airflow Core Provider +state: ready +# note that those versions are maintained by release manager - do not update them manually +versions: + - 1.0.0 + +dependencies: + - apache-airflow>=2.8.0 + +operators: + - integration-name: Core + python-modules: + - airflow.providers.core.time.operators.datetime + - airflow.providers.core.time.operators.weekday + +sensors: + - integration-name: Core + python-modules: + - airflow.providers.core.time.sensors.date_time + - airflow.providers.core.time.sensors.time_delta + - airflow.providers.core.time.sensors.time_sensor + - airflow.providers.core.time.sensors.weekday diff --git a/airflow/providers/core/time/__init__.py b/airflow/providers/core/time/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/core/time/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/core/time/operators/__init__.py b/airflow/providers/core/time/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/core/time/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/core/time/sensors/__init__.py b/airflow/providers/core/time/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/core/time/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/core/time/sensors/weekday.py b/airflow/providers/core/time/sensors/weekday.py new file mode 100644 index 0000000000000..b3e873b70e987 --- /dev/null +++ b/airflow/providers/core/time/sensors/weekday.py @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING, Iterable + +from airflow.sensors.base import BaseSensorOperator +from airflow.utils import timezone +from airflow.utils.weekday import WeekDay + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class DayOfWeekSensor(BaseSensorOperator): + """ + Waits until the first specified day of the week. + + For example, if the execution day of the task is '2018-12-22' (Saturday) + and you pass 'FRIDAY', the task will wait until next Friday. + + **Example** (with single day): :: + + weekend_check = DayOfWeekSensor( + task_id="weekend_check", week_day="Saturday", use_task_logical_date=True, dag=dag + ) + + **Example** (with multiple day using set): :: + + weekend_check = DayOfWeekSensor( + task_id="weekend_check", week_day={"Saturday", "Sunday"}, use_task_logical_date=True, dag=dag + ) + + **Example** (with :class:`~airflow.utils.weekday.WeekDay` enum): :: + + # import WeekDay Enum + from airflow.utils.weekday import WeekDay + + weekend_check = DayOfWeekSensor( + task_id="weekend_check", + week_day={WeekDay.SATURDAY, WeekDay.SUNDAY}, + use_task_logical_date=True, + dag=dag, + ) + + :param week_day: Day of the week to check (full name). Optionally, a set + of days can also be provided using a set. + Example values: + + * ``"MONDAY"``, + * ``{"Saturday", "Sunday"}`` + * ``{WeekDay.TUESDAY}`` + * ``{WeekDay.SATURDAY, WeekDay.SUNDAY}`` + + To use ``WeekDay`` enum, import it from ``airflow.utils.weekday`` + + :param use_task_logical_date: If ``True``, uses task's logical date to compare + with week_day. Execution Date is Useful for backfilling. + If ``False``, uses system's day of the week. Useful when you + don't want to run anything on weekdays on the system. + + .. seealso:: + For more information on how to use this sensor, take a look at the guide: + :ref:`howto/operator:DayOfWeekSensor` + + """ + + def __init__( + self, + *, + week_day: str | Iterable[str] | WeekDay | Iterable[WeekDay], + use_task_logical_date: bool = False, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.week_day = week_day + self.use_task_logical_date = use_task_logical_date + self._week_day_num = WeekDay.validate_week_day(week_day) + + def poke(self, context: Context) -> bool: + self.log.info( + "Poking until weekday is in %s, Today is %s", + self.week_day, + WeekDay(timezone.utcnow().isoweekday()).name, + ) + if self.use_task_logical_date: + return context["logical_date"].isoweekday() in self._week_day_num + else: + return timezone.utcnow().isoweekday() in self._week_day_num diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index db1315caea441..caa133eca1b8b 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -0aa10fcd2a9fcdaf15f0b207d71e3848 +86e09f3398f9cccdec52d9a80e421bfe diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 20e2e14faefed..04ab90d87f6b5 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -6953cb3aa552b67616179e1acfe7e8c3 +4f7e194639b897e1413eba454aea79d5 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt index c40492220b2a0..e0588bed2fe73 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt @@ -1 +1 @@ -b3d53d1c7e5233cebb3e35afed6ed8e6 +50d133bb95a9459eec0037de97920a4d diff --git a/docs/apache-airflow-providers-core/changelog.rst b/docs/apache-airflow-providers-core/changelog.rst new file mode 100644 index 0000000000000..236354518fa8a --- /dev/null +++ b/docs/apache-airflow-providers-core/changelog.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../airflow/providers/core/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-core/commits.rst b/docs/apache-airflow-providers-core/commits.rst new file mode 100644 index 0000000000000..1e9802470054a --- /dev/null +++ b/docs/apache-airflow-providers-core/commits.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. THIS FILE IS UPDATED AUTOMATICALLY_AT_RELEASE_TIME diff --git a/docs/apache-airflow-providers-core/index.rst b/docs/apache-airflow-providers-core/index.rst new file mode 100644 index 0000000000000..723027ada679e --- /dev/null +++ b/docs/apache-airflow-providers-core/index.rst @@ -0,0 +1,97 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-core`` +=========================================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/providers/core/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection types + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + Example DAGs + PyPI Repository + Installing from sources + Python API <_api/airflow/providers/core/index> + + + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-core package +------------------------------------------------------ + + +Release: 1.0.0 + +Provider package +---------------- + +This package is for the ``core`` provider. +All classes for this package are included in the ``airflow.providers.core`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation via +``pip install apache-airflow-providers-core``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider package is ``2.7.0``. + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=2.7.0`` +================== ================== diff --git a/docs/apache-airflow-providers-core/installing-providers-from-sources.rst b/docs/apache-airflow-providers-core/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..b4e730f4ff21a --- /dev/null +++ b/docs/apache-airflow-providers-core/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-core/security.rst b/docs/apache-airflow-providers-core/security.rst new file mode 100644 index 0000000000000..afa13dac6fc9b --- /dev/null +++ b/docs/apache-airflow-providers-core/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/security.rst diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index b1ff0f7dcfab7..e932e16d69a67 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -428,6 +428,16 @@ "excluded-python-versions": [], "state": "ready" }, + "core": { + "deps": [ + "apache-airflow>=2.8.0" + ], + "devel-deps": [], + "plugins": [], + "cross-providers-deps": [], + "excluded-python-versions": [], + "state": "ready" + }, "databricks": { "deps": [ "aiohttp>=3.9.2, <4", diff --git a/tests/providers/core/__init__.py b/tests/providers/core/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/core/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/core/time/__init__.py b/tests/providers/core/time/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/core/time/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/core/time/operators/__init__.py b/tests/providers/core/time/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/core/time/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/core/time/sensors/__init__.py b/tests/providers/core/time/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/core/time/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/sensors/test_external_task_sensor.py b/tests/providers/core/time/sensors/test_external_task_sensor.py similarity index 100% rename from tests/sensors/test_external_task_sensor.py rename to tests/providers/core/time/sensors/test_external_task_sensor.py diff --git a/tests/system/providers/core/__init__.py b/tests/system/providers/core/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/core/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/core/time/__init__.py b/tests/system/providers/core/time/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/core/time/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/core/time/operators/__init__.py b/tests/system/providers/core/time/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/core/time/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/example_dags/example_branch_datetime_operator.py b/tests/system/providers/core/time/operators/example_branch_datetime_operator.py similarity index 100% rename from airflow/example_dags/example_branch_datetime_operator.py rename to tests/system/providers/core/time/operators/example_branch_datetime_operator.py diff --git a/airflow/example_dags/example_branch_day_of_week_operator.py b/tests/system/providers/core/time/operators/example_branch_day_of_week_operator.py similarity index 100% rename from airflow/example_dags/example_branch_day_of_week_operator.py rename to tests/system/providers/core/time/operators/example_branch_day_of_week_operator.py diff --git a/tests/system/providers/core/time/sensors/__init__.py b/tests/system/providers/core/time/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/core/time/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/example_dags/example_sensors.py b/tests/system/providers/core/time/sensors/example_sensors.py similarity index 100% rename from airflow/example_dags/example_sensors.py rename to tests/system/providers/core/time/sensors/example_sensors.py diff --git a/airflow/example_dags/example_time_delta_sensor_async.py b/tests/system/providers/core/time/sensors/example_time_delta_sensor_async.py similarity index 100% rename from airflow/example_dags/example_time_delta_sensor_async.py rename to tests/system/providers/core/time/sensors/example_time_delta_sensor_async.py From e8ca29534b83871268d97359dceb7eae1719c4ac Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Mon, 19 Aug 2024 07:48:37 +0300 Subject: [PATCH 02/48] add source-date-epoch --- airflow/providers/core/provider.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airflow/providers/core/provider.yaml b/airflow/providers/core/provider.yaml index a535f4a75652c..01b9ff08f69da 100644 --- a/airflow/providers/core/provider.yaml +++ b/airflow/providers/core/provider.yaml @@ -21,6 +21,8 @@ name: Core description: | Airflow Core Provider state: ready +source-date-epoch: 1718603992 + # note that those versions are maintained by release manager - do not update them manually versions: - 1.0.0 From 7145726457e7e1d4867a388f399d0dc1a51bf383 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Mon, 19 Aug 2024 23:55:22 +0300 Subject: [PATCH 03/48] change core to essentials --- .pre-commit-config.yaml | 2 +- .../{core => essentials}/CHANGELOG.rst | 0 .../{core => essentials}/__init__.py | 0 .../{core => essentials}/provider.yaml | 22 +++++++++---------- .../{core => essentials}/time/__init__.py | 0 .../time/operators/__init__.py | 0 .../time/sensors/__init__.py | 0 .../time/sensors/weekday.py | 0 ...ement_generate-issue-content-providers.txt | 2 +- ...agement_prepare-provider-documentation.txt | 2 +- ...e-management_prepare-provider-packages.txt | 2 +- .../changelog.rst | 2 +- .../commits.rst | 0 .../index.rst | 22 +++++++++---------- .../installing-providers-from-sources.rst | 0 .../security.rst | 0 generated/provider_dependencies.json | 20 ++++++++--------- .../{core => essentials}/__init__.py | 0 .../{core => essentials}/time/__init__.py | 0 .../time/operators/__init__.py | 0 .../time/sensors/__init__.py | 0 .../time/sensors/test_external_task_sensor.py | 0 .../{core => essentials}/__init__.py | 0 .../{core => essentials}/time/__init__.py | 0 .../time/operators/__init__.py | 0 .../example_branch_datetime_operator.py | 0 .../example_branch_day_of_week_operator.py | 0 .../time/sensors/__init__.py | 0 .../time/sensors/example_sensors.py | 0 .../example_time_delta_sensor_async.py | 0 30 files changed, 37 insertions(+), 37 deletions(-) rename airflow/providers/{core => essentials}/CHANGELOG.rst (100%) rename airflow/providers/{core => essentials}/__init__.py (100%) rename airflow/providers/{core => essentials}/provider.yaml (66%) rename airflow/providers/{core => essentials}/time/__init__.py (100%) rename airflow/providers/{core => essentials}/time/operators/__init__.py (100%) rename airflow/providers/{core => essentials}/time/sensors/__init__.py (100%) rename airflow/providers/{core => essentials}/time/sensors/weekday.py (100%) rename docs/{apache-airflow-providers-core => apache-airflow-providers-essentials}/changelog.rst (92%) rename docs/{apache-airflow-providers-core => apache-airflow-providers-essentials}/commits.rst (100%) rename docs/{apache-airflow-providers-core => apache-airflow-providers-essentials}/index.rst (82%) rename docs/{apache-airflow-providers-core => apache-airflow-providers-essentials}/installing-providers-from-sources.rst (100%) rename docs/{apache-airflow-providers-core => apache-airflow-providers-essentials}/security.rst (100%) rename tests/providers/{core => essentials}/__init__.py (100%) rename tests/providers/{core => essentials}/time/__init__.py (100%) rename tests/providers/{core => essentials}/time/operators/__init__.py (100%) rename tests/providers/{core => essentials}/time/sensors/__init__.py (100%) rename tests/providers/{core => essentials}/time/sensors/test_external_task_sensor.py (100%) rename tests/system/providers/{core => essentials}/__init__.py (100%) rename tests/system/providers/{core => essentials}/time/__init__.py (100%) rename tests/system/providers/{core => essentials}/time/operators/__init__.py (100%) rename tests/system/providers/{core => essentials}/time/operators/example_branch_datetime_operator.py (100%) rename tests/system/providers/{core => essentials}/time/operators/example_branch_day_of_week_operator.py (100%) rename tests/system/providers/{core => essentials}/time/sensors/__init__.py (100%) rename tests/system/providers/{core => essentials}/time/sensors/example_sensors.py (100%) rename tests/system/providers/{core => essentials}/time/sensors/example_time_delta_sensor_async.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9f44da6afceb4..dc7d88cd12078 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -956,7 +956,7 @@ repos: exclude: > (?x) ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ | - ^tests/system/providers/core/time/operators/example_branch_datetime_operator.py + ^tests/system/providers/essentials/time/operators/example_branch_datetime_operator.py pass_filenames: true additional_dependencies: ['rich>=12.4.4'] - id: generate-pypi-readme diff --git a/airflow/providers/core/CHANGELOG.rst b/airflow/providers/essentials/CHANGELOG.rst similarity index 100% rename from airflow/providers/core/CHANGELOG.rst rename to airflow/providers/essentials/CHANGELOG.rst diff --git a/airflow/providers/core/__init__.py b/airflow/providers/essentials/__init__.py similarity index 100% rename from airflow/providers/core/__init__.py rename to airflow/providers/essentials/__init__.py diff --git a/airflow/providers/core/provider.yaml b/airflow/providers/essentials/provider.yaml similarity index 66% rename from airflow/providers/core/provider.yaml rename to airflow/providers/essentials/provider.yaml index 01b9ff08f69da..caef50d8f007d 100644 --- a/airflow/providers/core/provider.yaml +++ b/airflow/providers/essentials/provider.yaml @@ -16,10 +16,10 @@ # under the License. --- -package-name: apache-airflow-providers-core -name: Core +package-name: apache-airflow-providers-essentials +name: Essentials description: | - Airflow Core Provider + Airflow Essentials Provider state: ready source-date-epoch: 1718603992 @@ -31,15 +31,15 @@ dependencies: - apache-airflow>=2.8.0 operators: - - integration-name: Core + - integration-name: Essentials python-modules: - - airflow.providers.core.time.operators.datetime - - airflow.providers.core.time.operators.weekday + - airflow.providers.essentials.time.operators.datetime + - airflow.providers.essentials.time.operators.weekday sensors: - - integration-name: Core + - integration-name: Essentials python-modules: - - airflow.providers.core.time.sensors.date_time - - airflow.providers.core.time.sensors.time_delta - - airflow.providers.core.time.sensors.time_sensor - - airflow.providers.core.time.sensors.weekday + - airflow.providers.essentials.time.sensors.date_time + - airflow.providers.essentials.time.sensors.time_delta + - airflow.providers.essentials.time.sensors.time_sensor + - airflow.providers.essentials.time.sensors.weekday diff --git a/airflow/providers/core/time/__init__.py b/airflow/providers/essentials/time/__init__.py similarity index 100% rename from airflow/providers/core/time/__init__.py rename to airflow/providers/essentials/time/__init__.py diff --git a/airflow/providers/core/time/operators/__init__.py b/airflow/providers/essentials/time/operators/__init__.py similarity index 100% rename from airflow/providers/core/time/operators/__init__.py rename to airflow/providers/essentials/time/operators/__init__.py diff --git a/airflow/providers/core/time/sensors/__init__.py b/airflow/providers/essentials/time/sensors/__init__.py similarity index 100% rename from airflow/providers/core/time/sensors/__init__.py rename to airflow/providers/essentials/time/sensors/__init__.py diff --git a/airflow/providers/core/time/sensors/weekday.py b/airflow/providers/essentials/time/sensors/weekday.py similarity index 100% rename from airflow/providers/core/time/sensors/weekday.py rename to airflow/providers/essentials/time/sensors/weekday.py diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index caa133eca1b8b..dac4d2e733798 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -86e09f3398f9cccdec52d9a80e421bfe +a3b07e58eba8eb3556d10141ab7cbf4a diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 04ab90d87f6b5..8ecd9bc10ec03 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -4f7e194639b897e1413eba454aea79d5 +cb0cc1e75dc7ef5d3b0b6ef76e6e9705 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt index e0588bed2fe73..b7f6d5f0b227e 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt @@ -1 +1 @@ -50d133bb95a9459eec0037de97920a4d +8ef690a3d1695e07d26fe60c4e4e3bba diff --git a/docs/apache-airflow-providers-core/changelog.rst b/docs/apache-airflow-providers-essentials/changelog.rst similarity index 92% rename from docs/apache-airflow-providers-core/changelog.rst rename to docs/apache-airflow-providers-essentials/changelog.rst index 236354518fa8a..b201a423c2eae 100644 --- a/docs/apache-airflow-providers-core/changelog.rst +++ b/docs/apache-airflow-providers-essentials/changelog.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../../airflow/providers/core/CHANGELOG.rst +.. include:: ../../airflow/providers/essentials/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-core/commits.rst b/docs/apache-airflow-providers-essentials/commits.rst similarity index 100% rename from docs/apache-airflow-providers-core/commits.rst rename to docs/apache-airflow-providers-essentials/commits.rst diff --git a/docs/apache-airflow-providers-core/index.rst b/docs/apache-airflow-providers-essentials/index.rst similarity index 82% rename from docs/apache-airflow-providers-core/index.rst rename to docs/apache-airflow-providers-essentials/index.rst index 723027ada679e..1d84d4338b261 100644 --- a/docs/apache-airflow-providers-core/index.rst +++ b/docs/apache-airflow-providers-essentials/index.rst @@ -16,7 +16,7 @@ specific language governing permissions and limitations under the License. -``apache-airflow-providers-core`` +``apache-airflow-providers-essentials`` =========================================== @@ -34,7 +34,7 @@ :maxdepth: 1 :caption: System tests - System Tests <_api/tests/system/providers/core/index> + System Tests <_api/tests/system/providers/essentials/index> .. toctree:: :hidden: @@ -48,10 +48,10 @@ :maxdepth: 1 :caption: Resources - Example DAGs - PyPI Repository + Example DAGs + PyPI Repository Installing from sources - Python API <_api/airflow/providers/core/index> + Python API <_api/airflow/providers/essentials/index> @@ -66,7 +66,7 @@ Detailed list of commits -apache-airflow-providers-core package +apache-airflow-providers-essentials package ------------------------------------------------------ @@ -75,23 +75,23 @@ Release: 1.0.0 Provider package ---------------- -This package is for the ``core`` provider. -All classes for this package are included in the ``airflow.providers.core`` python package. +This package is for the ``essentials`` provider. +All classes for this package are included in the ``airflow.providers.essentials`` python package. Installation ------------ You can install this package on top of an existing Airflow 2 installation via -``pip install apache-airflow-providers-core``. +``pip install apache-airflow-providers-essentials``. For the minimum Airflow version supported, see ``Requirements`` below. Requirements ------------ -The minimum Apache Airflow version supported by this provider package is ``2.7.0``. +The minimum Apache Airflow version supported by this provider package is ``2.8.0``. ================== ================== PIP package Version required ================== ================== -``apache-airflow`` ``>=2.7.0`` +``apache-airflow`` ``>=2.8.0`` ================== ================== diff --git a/docs/apache-airflow-providers-core/installing-providers-from-sources.rst b/docs/apache-airflow-providers-essentials/installing-providers-from-sources.rst similarity index 100% rename from docs/apache-airflow-providers-core/installing-providers-from-sources.rst rename to docs/apache-airflow-providers-essentials/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-core/security.rst b/docs/apache-airflow-providers-essentials/security.rst similarity index 100% rename from docs/apache-airflow-providers-core/security.rst rename to docs/apache-airflow-providers-essentials/security.rst diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index e932e16d69a67..a794cd25d3d69 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -428,16 +428,6 @@ "excluded-python-versions": [], "state": "ready" }, - "core": { - "deps": [ - "apache-airflow>=2.8.0" - ], - "devel-deps": [], - "plugins": [], - "cross-providers-deps": [], - "excluded-python-versions": [], - "state": "ready" - }, "databricks": { "deps": [ "aiohttp>=3.9.2, <4", @@ -560,6 +550,16 @@ "excluded-python-versions": [], "state": "ready" }, + "essentials": { + "deps": [ + "apache-airflow>=2.8.0" + ], + "devel-deps": [], + "plugins": [], + "cross-providers-deps": [], + "excluded-python-versions": [], + "state": "ready" + }, "exasol": { "deps": [ "apache-airflow-providers-common-sql>=1.14.1", diff --git a/tests/providers/core/__init__.py b/tests/providers/essentials/__init__.py similarity index 100% rename from tests/providers/core/__init__.py rename to tests/providers/essentials/__init__.py diff --git a/tests/providers/core/time/__init__.py b/tests/providers/essentials/time/__init__.py similarity index 100% rename from tests/providers/core/time/__init__.py rename to tests/providers/essentials/time/__init__.py diff --git a/tests/providers/core/time/operators/__init__.py b/tests/providers/essentials/time/operators/__init__.py similarity index 100% rename from tests/providers/core/time/operators/__init__.py rename to tests/providers/essentials/time/operators/__init__.py diff --git a/tests/providers/core/time/sensors/__init__.py b/tests/providers/essentials/time/sensors/__init__.py similarity index 100% rename from tests/providers/core/time/sensors/__init__.py rename to tests/providers/essentials/time/sensors/__init__.py diff --git a/tests/providers/core/time/sensors/test_external_task_sensor.py b/tests/providers/essentials/time/sensors/test_external_task_sensor.py similarity index 100% rename from tests/providers/core/time/sensors/test_external_task_sensor.py rename to tests/providers/essentials/time/sensors/test_external_task_sensor.py diff --git a/tests/system/providers/core/__init__.py b/tests/system/providers/essentials/__init__.py similarity index 100% rename from tests/system/providers/core/__init__.py rename to tests/system/providers/essentials/__init__.py diff --git a/tests/system/providers/core/time/__init__.py b/tests/system/providers/essentials/time/__init__.py similarity index 100% rename from tests/system/providers/core/time/__init__.py rename to tests/system/providers/essentials/time/__init__.py diff --git a/tests/system/providers/core/time/operators/__init__.py b/tests/system/providers/essentials/time/operators/__init__.py similarity index 100% rename from tests/system/providers/core/time/operators/__init__.py rename to tests/system/providers/essentials/time/operators/__init__.py diff --git a/tests/system/providers/core/time/operators/example_branch_datetime_operator.py b/tests/system/providers/essentials/time/operators/example_branch_datetime_operator.py similarity index 100% rename from tests/system/providers/core/time/operators/example_branch_datetime_operator.py rename to tests/system/providers/essentials/time/operators/example_branch_datetime_operator.py diff --git a/tests/system/providers/core/time/operators/example_branch_day_of_week_operator.py b/tests/system/providers/essentials/time/operators/example_branch_day_of_week_operator.py similarity index 100% rename from tests/system/providers/core/time/operators/example_branch_day_of_week_operator.py rename to tests/system/providers/essentials/time/operators/example_branch_day_of_week_operator.py diff --git a/tests/system/providers/core/time/sensors/__init__.py b/tests/system/providers/essentials/time/sensors/__init__.py similarity index 100% rename from tests/system/providers/core/time/sensors/__init__.py rename to tests/system/providers/essentials/time/sensors/__init__.py diff --git a/tests/system/providers/core/time/sensors/example_sensors.py b/tests/system/providers/essentials/time/sensors/example_sensors.py similarity index 100% rename from tests/system/providers/core/time/sensors/example_sensors.py rename to tests/system/providers/essentials/time/sensors/example_sensors.py diff --git a/tests/system/providers/core/time/sensors/example_time_delta_sensor_async.py b/tests/system/providers/essentials/time/sensors/example_time_delta_sensor_async.py similarity index 100% rename from tests/system/providers/core/time/sensors/example_time_delta_sensor_async.py rename to tests/system/providers/essentials/time/sensors/example_time_delta_sensor_async.py From c251c000f9b8b1386c98eac444db394229071fff Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Tue, 20 Aug 2024 00:16:15 +0300 Subject: [PATCH 04/48] revert external task sensor location --- .../essentials/time => }/sensors/test_external_task_sensor.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{providers/essentials/time => }/sensors/test_external_task_sensor.py (100%) diff --git a/tests/providers/essentials/time/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py similarity index 100% rename from tests/providers/essentials/time/sensors/test_external_task_sensor.py rename to tests/sensors/test_external_task_sensor.py From a5f5a57e655299a2e4f74d816dbe20a68dcb7ace Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Wed, 21 Aug 2024 11:40:37 +0300 Subject: [PATCH 05/48] add provider to airflow_providers_bug_report list --- .github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index c5051297140ef..83ea64f1dfbd8 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -52,7 +52,6 @@ body: - common-compat - common-io - common-sql - - core - databricks - datadog - dbt-cloud @@ -61,6 +60,7 @@ body: - docker - edge - elasticsearch + - essentials - exasol - fab - facebook From 963a6aa11afab101402c45d4c4704c56da6d7207 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Sun, 25 Aug 2024 08:54:31 +0300 Subject: [PATCH 06/48] change new provider name to standard --- .../airflow_providers_bug_report.yml | 1 - .pre-commit-config.yaml | 2 +- airflow/providers/essentials/CHANGELOG.rst | 38 ------- airflow/providers/essentials/__init__.py | 17 --- airflow/providers/essentials/provider.yaml | 45 -------- .../essentials/time/sensors/weekday.py | 104 ------------------ ...ement_generate-issue-content-providers.txt | 2 +- ...agement_prepare-provider-documentation.txt | 2 +- ...e-management_prepare-provider-packages.txt | 2 +- .../changelog.rst | 18 --- .../commits.rst | 18 --- .../index.rst | 97 ---------------- .../installing-providers-from-sources.rst | 18 --- .../security.rst | 18 --- tests/providers/essentials/time/__init__.py | 16 --- .../essentials/time/operators/__init__.py | 16 --- .../essentials/time/sensors/__init__.py | 16 --- tests/system/providers/essentials/__init__.py | 16 --- .../providers/essentials/time/__init__.py | 16 --- .../essentials/time/operators/__init__.py | 16 --- .../essentials/time/sensors/__init__.py | 16 --- .../system/providers/standard}/__init__.py | 0 .../providers/standard/time}/__init__.py | 0 .../standard/time/operators}/__init__.py | 0 .../example_branch_datetime_operator.py | 0 .../example_branch_day_of_week_operator.py | 0 .../standard/time/sensors}/__init__.py | 0 .../time/sensors/example_sensors.py | 0 .../example_time_delta_sensor_async.py | 0 29 files changed, 4 insertions(+), 490 deletions(-) delete mode 100644 airflow/providers/essentials/CHANGELOG.rst delete mode 100644 airflow/providers/essentials/__init__.py delete mode 100644 airflow/providers/essentials/provider.yaml delete mode 100644 airflow/providers/essentials/time/sensors/weekday.py delete mode 100644 docs/apache-airflow-providers-essentials/changelog.rst delete mode 100644 docs/apache-airflow-providers-essentials/commits.rst delete mode 100644 docs/apache-airflow-providers-essentials/index.rst delete mode 100644 docs/apache-airflow-providers-essentials/installing-providers-from-sources.rst delete mode 100644 docs/apache-airflow-providers-essentials/security.rst delete mode 100644 tests/providers/essentials/time/__init__.py delete mode 100644 tests/providers/essentials/time/operators/__init__.py delete mode 100644 tests/providers/essentials/time/sensors/__init__.py delete mode 100644 tests/system/providers/essentials/__init__.py delete mode 100644 tests/system/providers/essentials/time/__init__.py delete mode 100644 tests/system/providers/essentials/time/operators/__init__.py delete mode 100644 tests/system/providers/essentials/time/sensors/__init__.py rename {airflow/providers/essentials/time => tests/system/providers/standard}/__init__.py (100%) rename {airflow/providers/essentials/time/operators => tests/system/providers/standard/time}/__init__.py (100%) rename {airflow/providers/essentials/time/sensors => tests/system/providers/standard/time/operators}/__init__.py (100%) rename tests/system/providers/{essentials => standard}/time/operators/example_branch_datetime_operator.py (100%) rename tests/system/providers/{essentials => standard}/time/operators/example_branch_day_of_week_operator.py (100%) rename tests/{providers/essentials => system/providers/standard/time/sensors}/__init__.py (100%) rename tests/system/providers/{essentials => standard}/time/sensors/example_sensors.py (100%) rename tests/system/providers/{essentials => standard}/time/sensors/example_time_delta_sensor_async.py (100%) diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index 83ea64f1dfbd8..03a19e8c7753e 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -60,7 +60,6 @@ body: - docker - edge - elasticsearch - - essentials - exasol - fab - facebook diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc7d88cd12078..491ee435d5ac8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -956,7 +956,7 @@ repos: exclude: > (?x) ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ | - ^tests/system/providers/essentials/time/operators/example_branch_datetime_operator.py + ^tests/system/providers/standard/time/operators/example_branch_datetime_operator.py pass_filenames: true additional_dependencies: ['rich>=12.4.4'] - id: generate-pypi-readme diff --git a/airflow/providers/essentials/CHANGELOG.rst b/airflow/providers/essentials/CHANGELOG.rst deleted file mode 100644 index 85db68915f4f2..0000000000000 --- a/airflow/providers/essentials/CHANGELOG.rst +++ /dev/null @@ -1,38 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. http://www.apache.org/licenses/LICENSE-2.0 - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -.. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes - and you want to add an explanation to the users on how they are supposed to deal with them. - The changelog is updated and maintained semi-automatically by release manager. -``apache-airflow-providers-core`` - - -Changelog ---------- - -1.0.0 -..... - -Initial version of the provider. diff --git a/airflow/providers/essentials/__init__.py b/airflow/providers/essentials/__init__.py deleted file mode 100644 index 217e5db960782..0000000000000 --- a/airflow/providers/essentials/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/essentials/provider.yaml b/airflow/providers/essentials/provider.yaml deleted file mode 100644 index caef50d8f007d..0000000000000 --- a/airflow/providers/essentials/provider.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -package-name: apache-airflow-providers-essentials -name: Essentials -description: | - Airflow Essentials Provider -state: ready -source-date-epoch: 1718603992 - -# note that those versions are maintained by release manager - do not update them manually -versions: - - 1.0.0 - -dependencies: - - apache-airflow>=2.8.0 - -operators: - - integration-name: Essentials - python-modules: - - airflow.providers.essentials.time.operators.datetime - - airflow.providers.essentials.time.operators.weekday - -sensors: - - integration-name: Essentials - python-modules: - - airflow.providers.essentials.time.sensors.date_time - - airflow.providers.essentials.time.sensors.time_delta - - airflow.providers.essentials.time.sensors.time_sensor - - airflow.providers.essentials.time.sensors.weekday diff --git a/airflow/providers/essentials/time/sensors/weekday.py b/airflow/providers/essentials/time/sensors/weekday.py deleted file mode 100644 index b3e873b70e987..0000000000000 --- a/airflow/providers/essentials/time/sensors/weekday.py +++ /dev/null @@ -1,104 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -from typing import TYPE_CHECKING, Iterable - -from airflow.sensors.base import BaseSensorOperator -from airflow.utils import timezone -from airflow.utils.weekday import WeekDay - -if TYPE_CHECKING: - from airflow.utils.context import Context - - -class DayOfWeekSensor(BaseSensorOperator): - """ - Waits until the first specified day of the week. - - For example, if the execution day of the task is '2018-12-22' (Saturday) - and you pass 'FRIDAY', the task will wait until next Friday. - - **Example** (with single day): :: - - weekend_check = DayOfWeekSensor( - task_id="weekend_check", week_day="Saturday", use_task_logical_date=True, dag=dag - ) - - **Example** (with multiple day using set): :: - - weekend_check = DayOfWeekSensor( - task_id="weekend_check", week_day={"Saturday", "Sunday"}, use_task_logical_date=True, dag=dag - ) - - **Example** (with :class:`~airflow.utils.weekday.WeekDay` enum): :: - - # import WeekDay Enum - from airflow.utils.weekday import WeekDay - - weekend_check = DayOfWeekSensor( - task_id="weekend_check", - week_day={WeekDay.SATURDAY, WeekDay.SUNDAY}, - use_task_logical_date=True, - dag=dag, - ) - - :param week_day: Day of the week to check (full name). Optionally, a set - of days can also be provided using a set. - Example values: - - * ``"MONDAY"``, - * ``{"Saturday", "Sunday"}`` - * ``{WeekDay.TUESDAY}`` - * ``{WeekDay.SATURDAY, WeekDay.SUNDAY}`` - - To use ``WeekDay`` enum, import it from ``airflow.utils.weekday`` - - :param use_task_logical_date: If ``True``, uses task's logical date to compare - with week_day. Execution Date is Useful for backfilling. - If ``False``, uses system's day of the week. Useful when you - don't want to run anything on weekdays on the system. - - .. seealso:: - For more information on how to use this sensor, take a look at the guide: - :ref:`howto/operator:DayOfWeekSensor` - - """ - - def __init__( - self, - *, - week_day: str | Iterable[str] | WeekDay | Iterable[WeekDay], - use_task_logical_date: bool = False, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.week_day = week_day - self.use_task_logical_date = use_task_logical_date - self._week_day_num = WeekDay.validate_week_day(week_day) - - def poke(self, context: Context) -> bool: - self.log.info( - "Poking until weekday is in %s, Today is %s", - self.week_day, - WeekDay(timezone.utcnow().isoweekday()).name, - ) - if self.use_task_logical_date: - return context["logical_date"].isoweekday() in self._week_day_num - else: - return timezone.utcnow().isoweekday() in self._week_day_num diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index dac4d2e733798..912f4a0d63891 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -a3b07e58eba8eb3556d10141ab7cbf4a +6806d4e405c59f8ee43796ae638b1308 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 8ecd9bc10ec03..551daf2a8ed1d 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -cb0cc1e75dc7ef5d3b0b6ef76e6e9705 +1baef77ed7f5328a75248bb91a2d3a11 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt index b7f6d5f0b227e..5caa5aa1be316 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt @@ -1 +1 @@ -8ef690a3d1695e07d26fe60c4e4e3bba +a781b53f55fe962ebab27068bcd96e44 diff --git a/docs/apache-airflow-providers-essentials/changelog.rst b/docs/apache-airflow-providers-essentials/changelog.rst deleted file mode 100644 index b201a423c2eae..0000000000000 --- a/docs/apache-airflow-providers-essentials/changelog.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../../airflow/providers/essentials/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-essentials/commits.rst b/docs/apache-airflow-providers-essentials/commits.rst deleted file mode 100644 index 1e9802470054a..0000000000000 --- a/docs/apache-airflow-providers-essentials/commits.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. THIS FILE IS UPDATED AUTOMATICALLY_AT_RELEASE_TIME diff --git a/docs/apache-airflow-providers-essentials/index.rst b/docs/apache-airflow-providers-essentials/index.rst deleted file mode 100644 index 1d84d4338b261..0000000000000 --- a/docs/apache-airflow-providers-essentials/index.rst +++ /dev/null @@ -1,97 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -``apache-airflow-providers-essentials`` -=========================================== - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Basics - - Home - Changelog - Security - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: System tests - - System Tests <_api/tests/system/providers/essentials/index> - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Guides - - Connection types - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Resources - - Example DAGs - PyPI Repository - Installing from sources - Python API <_api/airflow/providers/essentials/index> - - - -.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Commits - - Detailed list of commits - - -apache-airflow-providers-essentials package ------------------------------------------------------- - - -Release: 1.0.0 - -Provider package ----------------- - -This package is for the ``essentials`` provider. -All classes for this package are included in the ``airflow.providers.essentials`` python package. - -Installation ------------- - -You can install this package on top of an existing Airflow 2 installation via -``pip install apache-airflow-providers-essentials``. -For the minimum Airflow version supported, see ``Requirements`` below. - -Requirements ------------- - -The minimum Apache Airflow version supported by this provider package is ``2.8.0``. - -================== ================== -PIP package Version required -================== ================== -``apache-airflow`` ``>=2.8.0`` -================== ================== diff --git a/docs/apache-airflow-providers-essentials/installing-providers-from-sources.rst b/docs/apache-airflow-providers-essentials/installing-providers-from-sources.rst deleted file mode 100644 index b4e730f4ff21a..0000000000000 --- a/docs/apache-airflow-providers-essentials/installing-providers-from-sources.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-essentials/security.rst b/docs/apache-airflow-providers-essentials/security.rst deleted file mode 100644 index afa13dac6fc9b..0000000000000 --- a/docs/apache-airflow-providers-essentials/security.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../exts/includes/security.rst diff --git a/tests/providers/essentials/time/__init__.py b/tests/providers/essentials/time/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/providers/essentials/time/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/essentials/time/operators/__init__.py b/tests/providers/essentials/time/operators/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/providers/essentials/time/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/essentials/time/sensors/__init__.py b/tests/providers/essentials/time/sensors/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/providers/essentials/time/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/essentials/__init__.py b/tests/system/providers/essentials/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/essentials/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/essentials/time/__init__.py b/tests/system/providers/essentials/time/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/essentials/time/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/essentials/time/operators/__init__.py b/tests/system/providers/essentials/time/operators/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/essentials/time/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/essentials/time/sensors/__init__.py b/tests/system/providers/essentials/time/sensors/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/essentials/time/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/essentials/time/__init__.py b/tests/system/providers/standard/__init__.py similarity index 100% rename from airflow/providers/essentials/time/__init__.py rename to tests/system/providers/standard/__init__.py diff --git a/airflow/providers/essentials/time/operators/__init__.py b/tests/system/providers/standard/time/__init__.py similarity index 100% rename from airflow/providers/essentials/time/operators/__init__.py rename to tests/system/providers/standard/time/__init__.py diff --git a/airflow/providers/essentials/time/sensors/__init__.py b/tests/system/providers/standard/time/operators/__init__.py similarity index 100% rename from airflow/providers/essentials/time/sensors/__init__.py rename to tests/system/providers/standard/time/operators/__init__.py diff --git a/tests/system/providers/essentials/time/operators/example_branch_datetime_operator.py b/tests/system/providers/standard/time/operators/example_branch_datetime_operator.py similarity index 100% rename from tests/system/providers/essentials/time/operators/example_branch_datetime_operator.py rename to tests/system/providers/standard/time/operators/example_branch_datetime_operator.py diff --git a/tests/system/providers/essentials/time/operators/example_branch_day_of_week_operator.py b/tests/system/providers/standard/time/operators/example_branch_day_of_week_operator.py similarity index 100% rename from tests/system/providers/essentials/time/operators/example_branch_day_of_week_operator.py rename to tests/system/providers/standard/time/operators/example_branch_day_of_week_operator.py diff --git a/tests/providers/essentials/__init__.py b/tests/system/providers/standard/time/sensors/__init__.py similarity index 100% rename from tests/providers/essentials/__init__.py rename to tests/system/providers/standard/time/sensors/__init__.py diff --git a/tests/system/providers/essentials/time/sensors/example_sensors.py b/tests/system/providers/standard/time/sensors/example_sensors.py similarity index 100% rename from tests/system/providers/essentials/time/sensors/example_sensors.py rename to tests/system/providers/standard/time/sensors/example_sensors.py diff --git a/tests/system/providers/essentials/time/sensors/example_time_delta_sensor_async.py b/tests/system/providers/standard/time/sensors/example_time_delta_sensor_async.py similarity index 100% rename from tests/system/providers/essentials/time/sensors/example_time_delta_sensor_async.py rename to tests/system/providers/standard/time/sensors/example_time_delta_sensor_async.py From 01592e0fed6287a5566f7cd85343994602c8551f Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Sun, 25 Aug 2024 14:15:31 +0300 Subject: [PATCH 07/48] add integration --- airflow/providers/standard/provider.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/providers/standard/provider.yaml b/airflow/providers/standard/provider.yaml index 83d8acf0a68b3..7ac5151efc6fb 100644 --- a/airflow/providers/standard/provider.yaml +++ b/airflow/providers/standard/provider.yaml @@ -37,6 +37,11 @@ integrations: how-to-guide: - /docs/apache-airflow-providers-standard/operators.rst +integrations: + - integration-name: Standard + external-doc-url: https://airflow.apache.org/ + tags: [apache] + operators: - integration-name: Standard python-modules: From 2fce0ffa1e7eac7e95d9e284f2626e90da37774e Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Sun, 25 Aug 2024 14:55:27 +0300 Subject: [PATCH 08/48] revert hatch_build --- hatch_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 72f59c79ef9ff..661bc35b13d13 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -49,7 +49,6 @@ "imap", "smtp", "sqlite", - "standard", ] # Those extras are dynamically added by hatch in the build hook to metadata optional dependencies From 8d961a31f4d56c1df0fb947b13c4b9872e869983 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Sun, 25 Aug 2024 22:54:00 +0300 Subject: [PATCH 09/48] move examples back to airflow core --- .../example_branch_datetime_operator.py | 0 .../example_branch_day_of_week_operator.py | 0 .../example_dags/sensors}/__init__.py | 0 .../example_dags}/sensors/example_sensors.py | 0 .../sensors/example_time_delta_sensor_async.py | 0 tests/system/providers/standard/time/__init__.py | 16 ---------------- .../standard/time/operators/__init__.py | 16 ---------------- .../providers/standard/time/sensors/__init__.py | 16 ---------------- 8 files changed, 48 deletions(-) rename {tests/system/providers/standard/time/operators => airflow/example_dags}/example_branch_datetime_operator.py (100%) rename {tests/system/providers/standard/time/operators => airflow/example_dags}/example_branch_day_of_week_operator.py (100%) rename {tests/system/providers/standard => airflow/example_dags/sensors}/__init__.py (100%) rename {tests/system/providers/standard/time => airflow/example_dags}/sensors/example_sensors.py (100%) rename {tests/system/providers/standard/time => airflow/example_dags}/sensors/example_time_delta_sensor_async.py (100%) delete mode 100644 tests/system/providers/standard/time/__init__.py delete mode 100644 tests/system/providers/standard/time/operators/__init__.py delete mode 100644 tests/system/providers/standard/time/sensors/__init__.py diff --git a/tests/system/providers/standard/time/operators/example_branch_datetime_operator.py b/airflow/example_dags/example_branch_datetime_operator.py similarity index 100% rename from tests/system/providers/standard/time/operators/example_branch_datetime_operator.py rename to airflow/example_dags/example_branch_datetime_operator.py diff --git a/tests/system/providers/standard/time/operators/example_branch_day_of_week_operator.py b/airflow/example_dags/example_branch_day_of_week_operator.py similarity index 100% rename from tests/system/providers/standard/time/operators/example_branch_day_of_week_operator.py rename to airflow/example_dags/example_branch_day_of_week_operator.py diff --git a/tests/system/providers/standard/__init__.py b/airflow/example_dags/sensors/__init__.py similarity index 100% rename from tests/system/providers/standard/__init__.py rename to airflow/example_dags/sensors/__init__.py diff --git a/tests/system/providers/standard/time/sensors/example_sensors.py b/airflow/example_dags/sensors/example_sensors.py similarity index 100% rename from tests/system/providers/standard/time/sensors/example_sensors.py rename to airflow/example_dags/sensors/example_sensors.py diff --git a/tests/system/providers/standard/time/sensors/example_time_delta_sensor_async.py b/airflow/example_dags/sensors/example_time_delta_sensor_async.py similarity index 100% rename from tests/system/providers/standard/time/sensors/example_time_delta_sensor_async.py rename to airflow/example_dags/sensors/example_time_delta_sensor_async.py diff --git a/tests/system/providers/standard/time/__init__.py b/tests/system/providers/standard/time/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/standard/time/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/standard/time/operators/__init__.py b/tests/system/providers/standard/time/operators/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/standard/time/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/system/providers/standard/time/sensors/__init__.py b/tests/system/providers/standard/time/sensors/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/system/providers/standard/time/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. From 40e07f186b418d94e6080f8ccd25d063fe39b6f3 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Mon, 26 Aug 2024 18:03:30 +0300 Subject: [PATCH 10/48] change sensors example dags paths --- docs/apache-airflow-providers-standard/sensors.rst | 8 ++++---- docs/apache-airflow/howto/operator/bash.rst | 2 +- docs/apache-airflow/howto/operator/file.rst | 4 ++-- docs/apache-airflow/howto/operator/python.rst | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/apache-airflow-providers-standard/sensors.rst b/docs/apache-airflow-providers-standard/sensors.rst index 77514dfbe86cf..98bec9e04e7f1 100644 --- a/docs/apache-airflow-providers-standard/sensors.rst +++ b/docs/apache-airflow-providers-standard/sensors.rst @@ -26,7 +26,7 @@ TimeDeltaSensor Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensor` to end sensing after specific time. -.. exampleinclude:: /../../airflow/example_dags/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor] @@ -42,7 +42,7 @@ Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensorAs It is an async version of the operator and requires Triggerer to run. -.. exampleinclude:: /../../airflow/example_dags/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor_async] @@ -57,7 +57,7 @@ TimeSensor Use the :class:`~airflow.providers.standard.sensors.time_sensor.TimeSensor` to end sensing after time specified. -.. exampleinclude:: /../../airflow/example_dags/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors] @@ -72,7 +72,7 @@ TimeSensorAsync Use the :class:`~airflow.providers.standard.sensors.time_sensor.TimeSensorAsync` to end sensing after time specified. It is an async version of the operator and requires Triggerer to run. -.. exampleinclude:: /../../airflow/example_dags/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors_async] diff --git a/docs/apache-airflow/howto/operator/bash.rst b/docs/apache-airflow/howto/operator/bash.rst index daf430fa14cde..905eca69ce355 100644 --- a/docs/apache-airflow/howto/operator/bash.rst +++ b/docs/apache-airflow/howto/operator/bash.rst @@ -393,7 +393,7 @@ BashSensor Use the :class:`~airflow.sensors.bash.BashSensor` to use arbitrary command for sensing. The command should return 0 when it succeeds, any other value otherwise. -.. exampleinclude:: /../../airflow/example_dags/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_bash_sensors] diff --git a/docs/apache-airflow/howto/operator/file.rst b/docs/apache-airflow/howto/operator/file.rst index 49ca1c75f6042..f3646da367077 100644 --- a/docs/apache-airflow/howto/operator/file.rst +++ b/docs/apache-airflow/howto/operator/file.rst @@ -26,7 +26,7 @@ Use the :class:`~airflow.sensors.filesystem.FileSensor` to detect files appearin filesystem. You need to have connection defined to use it (pass connection id via ``fs_conn_id``). Default connection is ``fs_default``. -.. exampleinclude:: /../../airflow/example_dags/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor] @@ -34,7 +34,7 @@ Default connection is ``fs_default``. Also for this job you can use sensor in the deferrable mode: -.. exampleinclude:: /../../airflow/example_dags/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor_async] diff --git a/docs/apache-airflow/howto/operator/python.rst b/docs/apache-airflow/howto/operator/python.rst index 2f0defddd886c..8b0a89653f789 100644 --- a/docs/apache-airflow/howto/operator/python.rst +++ b/docs/apache-airflow/howto/operator/python.rst @@ -566,7 +566,7 @@ value to be True. .. tab-item:: @task.sensor :sync: taskflow - .. exampleinclude:: /../../airflow/example_dags/example_sensor_decorator.py + .. exampleinclude:: /../../airflow/example_dags/sensors/example_sensor_decorator.py :language: python :dedent: 4 :start-after: [START wait_function] @@ -575,7 +575,7 @@ value to be True. .. tab-item:: PythonSensor :sync: operator - .. exampleinclude:: /../../airflow/example_dags/example_sensors.py + .. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py :language: python :dedent: 4 :start-after: [START example_python_sensors] From dd596a4e116bbf6bf9baa9e2bf8d89bf5ce39a33 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Mon, 26 Aug 2024 19:10:00 +0300 Subject: [PATCH 11/48] remove init --- airflow/example_dags/sensors/__init__.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 airflow/example_dags/sensors/__init__.py diff --git a/airflow/example_dags/sensors/__init__.py b/airflow/example_dags/sensors/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/airflow/example_dags/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. From 540a714bfd33def110457aec0d6e5d78e4c18071 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Mon, 26 Aug 2024 19:18:38 +0300 Subject: [PATCH 12/48] revert howto docs --- airflow/example_dags/{sensors => }/example_sensors.py | 0 .../{sensors => }/example_time_delta_sensor_async.py | 0 docs/apache-airflow-providers-standard/sensors.rst | 8 ++++---- docs/apache-airflow/howto/operator/bash.rst | 2 +- docs/apache-airflow/howto/operator/file.rst | 4 ++-- docs/apache-airflow/howto/operator/python.rst | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) rename airflow/example_dags/{sensors => }/example_sensors.py (100%) rename airflow/example_dags/{sensors => }/example_time_delta_sensor_async.py (100%) diff --git a/airflow/example_dags/sensors/example_sensors.py b/airflow/example_dags/example_sensors.py similarity index 100% rename from airflow/example_dags/sensors/example_sensors.py rename to airflow/example_dags/example_sensors.py diff --git a/airflow/example_dags/sensors/example_time_delta_sensor_async.py b/airflow/example_dags/example_time_delta_sensor_async.py similarity index 100% rename from airflow/example_dags/sensors/example_time_delta_sensor_async.py rename to airflow/example_dags/example_time_delta_sensor_async.py diff --git a/docs/apache-airflow-providers-standard/sensors.rst b/docs/apache-airflow-providers-standard/sensors.rst index 98bec9e04e7f1..77514dfbe86cf 100644 --- a/docs/apache-airflow-providers-standard/sensors.rst +++ b/docs/apache-airflow-providers-standard/sensors.rst @@ -26,7 +26,7 @@ TimeDeltaSensor Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensor` to end sensing after specific time. -.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor] @@ -42,7 +42,7 @@ Use the :class:`~airflow.providers.standard.sensors.time_delta.TimeDeltaSensorAs It is an async version of the operator and requires Triggerer to run. -.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_delta_sensor_async] @@ -57,7 +57,7 @@ TimeSensor Use the :class:`~airflow.providers.standard.sensors.time_sensor.TimeSensor` to end sensing after time specified. -.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors] @@ -72,7 +72,7 @@ TimeSensorAsync Use the :class:`~airflow.providers.standard.sensors.time_sensor.TimeSensorAsync` to end sensing after time specified. It is an async version of the operator and requires Triggerer to run. -.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_time_sensors_async] diff --git a/docs/apache-airflow/howto/operator/bash.rst b/docs/apache-airflow/howto/operator/bash.rst index 905eca69ce355..daf430fa14cde 100644 --- a/docs/apache-airflow/howto/operator/bash.rst +++ b/docs/apache-airflow/howto/operator/bash.rst @@ -393,7 +393,7 @@ BashSensor Use the :class:`~airflow.sensors.bash.BashSensor` to use arbitrary command for sensing. The command should return 0 when it succeeds, any other value otherwise. -.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_bash_sensors] diff --git a/docs/apache-airflow/howto/operator/file.rst b/docs/apache-airflow/howto/operator/file.rst index f3646da367077..49ca1c75f6042 100644 --- a/docs/apache-airflow/howto/operator/file.rst +++ b/docs/apache-airflow/howto/operator/file.rst @@ -26,7 +26,7 @@ Use the :class:`~airflow.sensors.filesystem.FileSensor` to detect files appearin filesystem. You need to have connection defined to use it (pass connection id via ``fs_conn_id``). Default connection is ``fs_default``. -.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor] @@ -34,7 +34,7 @@ Default connection is ``fs_default``. Also for this job you can use sensor in the deferrable mode: -.. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py +.. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_file_sensor_async] diff --git a/docs/apache-airflow/howto/operator/python.rst b/docs/apache-airflow/howto/operator/python.rst index 8b0a89653f789..2f0defddd886c 100644 --- a/docs/apache-airflow/howto/operator/python.rst +++ b/docs/apache-airflow/howto/operator/python.rst @@ -566,7 +566,7 @@ value to be True. .. tab-item:: @task.sensor :sync: taskflow - .. exampleinclude:: /../../airflow/example_dags/sensors/example_sensor_decorator.py + .. exampleinclude:: /../../airflow/example_dags/example_sensor_decorator.py :language: python :dedent: 4 :start-after: [START wait_function] @@ -575,7 +575,7 @@ value to be True. .. tab-item:: PythonSensor :sync: operator - .. exampleinclude:: /../../airflow/example_dags/sensors/example_sensors.py + .. exampleinclude:: /../../airflow/example_dags/example_sensors.py :language: python :dedent: 4 :start-after: [START example_python_sensors] From 5ca877231233942f556805df5388a495e9767cd6 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Sat, 7 Sep 2024 09:08:58 +0200 Subject: [PATCH 13/48] change provider as not-ready --- ...tput_release-management_generate-issue-content-providers.txt | 2 +- ...output_release-management_prepare-provider-documentation.txt | 2 +- .../output_release-management_prepare-provider-packages.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 912f4a0d63891..db1315caea441 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -6806d4e405c59f8ee43796ae638b1308 +0aa10fcd2a9fcdaf15f0b207d71e3848 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 551daf2a8ed1d..20e2e14faefed 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -1baef77ed7f5328a75248bb91a2d3a11 +6953cb3aa552b67616179e1acfe7e8c3 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt index 5caa5aa1be316..c40492220b2a0 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt @@ -1 +1 @@ -a781b53f55fe962ebab27068bcd96e44 +b3d53d1c7e5233cebb3e35afed6ed8e6 From 8cd744258b2086a6d67df3a8c25a8daf9d9d2636 Mon Sep 17 00:00:00 2001 From: romsharon98 Date: Sat, 14 Sep 2024 17:00:43 -0700 Subject: [PATCH 14/48] add changelog --- airflow/providers/standard/provider.yaml | 2 +- generated/provider_dependencies.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/standard/provider.yaml b/airflow/providers/standard/provider.yaml index 7ac5151efc6fb..cbc67aeb33c1f 100644 --- a/airflow/providers/standard/provider.yaml +++ b/airflow/providers/standard/provider.yaml @@ -28,7 +28,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.10.0 + - apache-airflow>=2.8.0 integrations: - integration-name: Standard diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index a794cd25d3d69..4158d2d95e622 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -1283,7 +1283,7 @@ }, "standard": { "deps": [ - "apache-airflow>=2.10.0" + "apache-airflow>=2.8.0" ], "devel-deps": [], "plugins": [], From e577f57d741d6699e10a0ffb19fc0b0c066eb1ca Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 16 Sep 2024 12:06:12 +0100 Subject: [PATCH 15/48] move bash operator and sensor to standard provider --- .pre-commit-config.yaml | 16 ++++++++++++++-- airflow/decorators/bash.py | 2 +- airflow/example_dags/example_bash_operator.py | 2 +- airflow/example_dags/example_complex.py | 2 +- airflow/example_dags/example_datasets.py | 0 .../example_dags/example_inlet_event_extra.py | 2 +- .../example_dags/example_outlet_event_extra.py | 2 +- .../example_passing_params_via_test_command.py | 2 +- airflow/example_dags/example_setup_teardown.py | 2 +- airflow/example_dags/example_task_group.py | 2 +- .../example_dags/example_trigger_target_dag.py | 2 +- airflow/example_dags/example_xcom.py | 2 +- airflow/example_dags/example_xcomargs.py | 2 +- airflow/example_dags/tutorial.py | 2 +- .../celery/executors/celery_executor_utils.py | 3 --- airflow/providers/openlineage/provider.yaml | 3 ++- airflow/providers/standard/core/__init__.py | 16 ++++++++++++++++ .../standard/core/operators/__init__.py | 16 ++++++++++++++++ .../standard/core}/operators/bash.py | 0 .../providers/standard/core/sensors/__init__.py | 16 ++++++++++++++++ .../standard/core}/sensors/bash.py | 0 dev/breeze/tests/test_selective_checks.py | 2 +- dev/perf/dags/elastic_dag.py | 2 +- dev/perf/dags/perf_dag_2.py | 2 +- .../notifications/chime_notifier_howto_guide.rst | 2 +- .../notifications/sns.rst | 2 +- .../notifications/sqs.rst | 2 +- .../apprise_notifier_howto_guide.rst | 2 +- .../notifications/jira-notifier-howto-guide.rst | 2 +- .../operators/cloud/mlengine.rst | 2 +- .../operators/cloud/pubsub.rst | 2 +- .../guides/developer.rst | 2 +- .../guides/user.rst | 4 ++-- .../pagerduty_notifier_howto_guide.rst | 2 +- .../notifications/slack_notifier_howto_guide.rst | 2 +- .../slackwebhook_notifier_howto_guide.rst | 2 +- .../notifications/smtp_notifier_howto_guide.rst | 2 +- .../administration-and-deployment/lineage.rst | 2 +- docs/apache-airflow/best-practices.rst | 2 +- docs/apache-airflow/core-concepts/dag-run.rst | 4 ++-- docs/apache-airflow/core-concepts/dags.rst | 2 +- docs/apache-airflow/core-concepts/operators.rst | 2 +- docs/apache-airflow/core-concepts/tasks.rst | 2 +- docs/apache-airflow/howto/notifications.rst | 2 +- docs/apache-airflow/howto/operator/bash.rst | 4 ++-- docs/apache-airflow/index.rst | 2 +- docs/apache-airflow/operators-and-hooks-ref.rst | 4 ++-- docs/apache-airflow/tutorial/taskflow.rst | 2 +- docs/exts/templates/openlineage.rst.jinja2 | 2 +- tests/callbacks/test_callback_requests.py | 6 +++--- tests/cli/commands/test_task_command.py | 2 +- tests/core/test_core.py | 2 +- tests/dags/subdir2/test_dont_ignore_this.py | 2 +- tests/dags/test_assets.py | 2 +- .../test_backfill_with_upstream_failed_task.py | 2 +- tests/dags/test_default_impersonation.py | 2 +- tests/dags/test_example_bash_operator.py | 2 +- tests/dags/test_failing.py | 2 +- tests/dags/test_heartbeat_failed_fast.py | 2 +- tests/dags/test_impersonation.py | 2 +- tests/dags/test_miscellaneous.py | 2 +- tests/dags/test_multiple_dags.py | 2 +- tests/dags/test_no_impersonation.py | 2 +- tests/dags/test_on_failure_callback.py | 2 +- tests/dags/test_retry_handling_job.py | 2 +- tests/decorators/test_setup_teardown.py | 2 +- .../executors/test_celery_executor.py | 2 +- tests/jobs/test_scheduler_job.py | 2 +- tests/listeners/test_listeners.py | 2 +- tests/models/test_dag.py | 2 +- tests/models/test_dagrun.py | 2 +- tests/models/test_renderedtifields.py | 2 +- tests/models/test_serialized_dag.py | 2 +- tests/models/test_taskinstance.py | 2 +- tests/models/test_xcom_arg.py | 2 +- .../executors/test_kubernetes_executor.py | 2 +- .../cncf/kubernetes/test_template_rendering.py | 2 +- .../openlineage/extractors/test_bash.py | 2 +- .../openlineage/extractors/test_python.py | 2 +- .../openlineage/plugins/test_adapter.py | 2 +- .../providers/openlineage/plugins/test_facets.py | 2 +- .../providers/openlineage/plugins/test_utils.py | 8 ++++---- tests/providers/openlineage/utils/test_utils.py | 14 +++++++------- tests/providers/standard/core/__init__.py | 16 ++++++++++++++++ .../standard/core/operators/__init__.py | 16 ++++++++++++++++ .../standard/core}/operators/test_bash.py | 2 +- .../providers/standard/core/sensors/__init__.py | 16 ++++++++++++++++ .../standard/core}/sensors/test_bash.py | 2 +- tests/serialization/test_dag_serialization.py | 8 ++++---- .../example_external_task_child_deferrable.py | 2 +- .../providers/amazon/aws/example_appflow.py | 2 +- .../providers/amazon/aws/example_http_to_s3.py | 2 +- tests/system/providers/amazon/aws/utils/k8s.py | 2 +- .../providers/apache/hive/example_twitter_dag.py | 2 +- .../providers/apache/iceberg/example_iceberg.py | 2 +- .../cncf/kubernetes/example_kubernetes.py | 2 +- .../cncf/kubernetes/example_kubernetes_async.py | 2 +- tests/system/providers/docker/example_docker.py | 2 +- .../providers/docker/example_docker_copy_data.py | 2 +- .../cloud/bigquery/example_bigquery_dataset.py | 2 +- .../cloud/bigquery/example_bigquery_queries.py | 2 +- .../bigquery/example_bigquery_queries_async.py | 2 +- .../cloud/bigquery/example_bigquery_to_mssql.py | 2 +- .../bigquery/example_bigquery_to_postgres.py | 2 +- .../cloud/cloud_build/example_cloud_build.py | 2 +- .../example_cloud_memorystore_memcached.py | 2 +- .../example_cloud_memorystore_redis.py | 2 +- .../google/cloud/gcs/example_gcs_copy_delete.py | 2 +- .../google/cloud/gcs/example_gcs_to_gcs.py | 2 +- .../google/cloud/gcs/example_mysql_to_gcs.py | 2 +- .../google/cloud/gcs/example_sftp_to_gcs.py | 2 +- .../providers/google/cloud/gcs/example_sheets.py | 2 +- .../example_kubernetes_engine.py | 2 +- .../example_kubernetes_engine_async.py | 2 +- .../google/cloud/ml_engine/example_mlengine.py | 2 +- .../natural_language/example_natural_language.py | 2 +- .../google/cloud/pubsub/example_pubsub.py | 2 +- .../cloud/sql_to_sheets/example_sql_to_sheets.py | 2 +- .../google/cloud/tasks/example_queue.py | 2 +- .../cloud/transfers/example_postgres_to_gcs.py | 2 +- .../google/cloud/translate/example_translate.py | 2 +- .../example_video_intelligence.py | 2 +- .../vision/example_vision_annotate_image.py | 2 +- .../datacatalog/example_datacatalog_entries.py | 2 +- .../example_datacatalog_search_catalog.py | 2 +- .../example_datacatalog_tag_templates.py | 2 +- .../datacatalog/example_datacatalog_tags.py | 2 +- .../opsgenie/example_opsgenie_notifier.py | 2 +- .../providers/singularity/example_singularity.py | 2 +- tests/utils/test_dot_renderer.py | 2 +- tests/utils/test_task_group.py | 2 +- tests/www/views/test_views_rendered.py | 2 +- tests/www/views/test_views_tasks.py | 2 +- 133 files changed, 251 insertions(+), 145 deletions(-) create mode 100644 airflow/example_dags/example_datasets.py create mode 100644 airflow/providers/standard/core/__init__.py create mode 100644 airflow/providers/standard/core/operators/__init__.py rename airflow/{ => providers/standard/core}/operators/bash.py (100%) create mode 100644 airflow/providers/standard/core/sensors/__init__.py rename airflow/{ => providers/standard/core}/sensors/bash.py (100%) create mode 100644 tests/providers/standard/core/__init__.py create mode 100644 tests/providers/standard/core/operators/__init__.py rename tests/{ => providers/standard/core}/operators/test_bash.py (99%) create mode 100644 tests/providers/standard/core/sensors/__init__.py rename tests/{ => providers/standard/core}/sensors/test_bash.py (97%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 491ee435d5ac8..e4d8886b55326 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -560,7 +560,19 @@ repos: ^airflow/example_dags/example_sensors.py| ^airflow/example_dags/example_sensors.py| ^airflow/example_dags/example_sensors.py| - ^airflow/example_dags/example_time_delta_sensor_async.py + ^airflow/example_dags/example_time_delta_sensor_async.py| + ^airflow/example_dags/example_bash_operator.py| + ^airflow/example_dags/example_complex.py| + ^airflow/example_dags/example_datasets.py| + ^airflow/example_dags/example_inlet_event_extra.py| + ^airflow/example_dags/example_outlet_event_extra.py| + ^airflow/example_dags/example_passing_params_via_test_command.py| + ^airflow/example_dags/example_setup_teardown.py| + ^airflow/example_dags/example_task_group.py| + ^airflow/example_dags/example_trigger_target_dag.py| + ^airflow/example_dags/example_xcom.py| + ^airflow/example_dags/example_xcomargs.py| + ^airflow/example_dags/tutorial.py files: ^airflow/example_dags/.*\.py$ - id: check-no-airflow-deprecation-in-providers language: pygrep @@ -717,7 +729,7 @@ repos: files: > (?x) ^airflow/providers/.*\.py$ - exclude: ^.*/.*_vendor/ + exclude: ^.*/.*_vendor/|airflow/providers/standard/core/operators/bash.py - id: check-get-lineage-collector-providers language: python name: Check providers import hook lineage code from compat diff --git a/airflow/decorators/bash.py b/airflow/decorators/bash.py index 39d3131d28c7b..5b6249859e948 100644 --- a/airflow/decorators/bash.py +++ b/airflow/decorators/bash.py @@ -21,7 +21,7 @@ from typing import Any, Callable, Collection, Mapping, Sequence from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.context import Context, context_merge from airflow.utils.operator_helpers import determine_kwargs from airflow.utils.types import NOTSET diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py index b08d31c9930c7..2883f6c2fb7f3 100644 --- a/airflow/example_dags/example_bash_operator.py +++ b/airflow/example_dags/example_bash_operator.py @@ -24,8 +24,8 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.core.operators.bash import BashOperator with DAG( dag_id="example_bash_operator", diff --git a/airflow/example_dags/example_complex.py b/airflow/example_dags/example_complex.py index e7eba78eae815..049d36c2ef120 100644 --- a/airflow/example_dags/example_complex.py +++ b/airflow/example_dags/example_complex.py @@ -25,7 +25,7 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator with DAG( dag_id="example_complex", diff --git a/airflow/example_dags/example_datasets.py b/airflow/example_dags/example_datasets.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airflow/example_dags/example_inlet_event_extra.py b/airflow/example_dags/example_inlet_event_extra.py index 974534c295b79..55fb24f16e190 100644 --- a/airflow/example_dags/example_inlet_event_extra.py +++ b/airflow/example_dags/example_inlet_event_extra.py @@ -28,7 +28,7 @@ from airflow.assets import Asset from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator asset = Asset("s3://output/1.txt") diff --git a/airflow/example_dags/example_outlet_event_extra.py b/airflow/example_dags/example_outlet_event_extra.py index 893090460b538..fc16809bfa4b7 100644 --- a/airflow/example_dags/example_outlet_event_extra.py +++ b/airflow/example_dags/example_outlet_event_extra.py @@ -29,7 +29,7 @@ from airflow.assets.metadata import Metadata from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator ds = Asset("s3://output/1.txt") diff --git a/airflow/example_dags/example_passing_params_via_test_command.py b/airflow/example_dags/example_passing_params_via_test_command.py index 2fcb8e4edab7b..3bdb9a25540a4 100644 --- a/airflow/example_dags/example_passing_params_via_test_command.py +++ b/airflow/example_dags/example_passing_params_via_test_command.py @@ -27,7 +27,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator @task(task_id="run_this") diff --git a/airflow/example_dags/example_setup_teardown.py b/airflow/example_dags/example_setup_teardown.py index 9fab87df7568b..b974931ba1b2f 100644 --- a/airflow/example_dags/example_setup_teardown.py +++ b/airflow/example_dags/example_setup_teardown.py @@ -22,7 +22,7 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup with DAG( diff --git a/airflow/example_dags/example_task_group.py b/airflow/example_dags/example_task_group.py index 6435a912cc419..487807a88f1f9 100644 --- a/airflow/example_dags/example_task_group.py +++ b/airflow/example_dags/example_task_group.py @@ -22,8 +22,8 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup # [START howto_task_group] diff --git a/airflow/example_dags/example_trigger_target_dag.py b/airflow/example_dags/example_trigger_target_dag.py index 7a009b8dcc6d1..793af39f12593 100644 --- a/airflow/example_dags/example_trigger_target_dag.py +++ b/airflow/example_dags/example_trigger_target_dag.py @@ -27,7 +27,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator @task(task_id="run_this") diff --git a/airflow/example_dags/example_xcom.py b/airflow/example_dags/example_xcom.py index fa99b91834658..ed6c2f427881b 100644 --- a/airflow/example_dags/example_xcom.py +++ b/airflow/example_dags/example_xcom.py @@ -24,7 +24,7 @@ from airflow.decorators import task from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator value_1 = [1, 2, 3] value_2 = {"a": "b"} diff --git a/airflow/example_dags/example_xcomargs.py b/airflow/example_dags/example_xcomargs.py index d9d0c94f4ea01..fd373c79dbd86 100644 --- a/airflow/example_dags/example_xcomargs.py +++ b/airflow/example_dags/example_xcomargs.py @@ -25,7 +25,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator log = logging.getLogger(__name__) diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py index 0e31775c7a9a7..707a668472ff8 100644 --- a/airflow/example_dags/tutorial.py +++ b/airflow/example_dags/tutorial.py @@ -32,7 +32,7 @@ from airflow.models.dag import DAG # Operators; we need this to operate! -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator # [END import_module] diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index 8f25f040c90ad..359b657c0f075 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -109,9 +109,6 @@ def on_celery_import_modules(*args, **kwargs): """ import jinja2.ext # noqa: F401 - import airflow.jobs.local_task_job_runner - import airflow.macros - import airflow.operators.bash import airflow.operators.python # noqa: F401 with contextlib.suppress(ImportError): diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index b249ff46c8591..dc67458b3f474 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -84,7 +84,8 @@ config: Exclude some Operators from emitting OpenLineage events by passing a string of semicolon separated full import paths of Operators to disable. type: string - example: "airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator" + example: "airflow.providers.standard.core.operators.bash.BashOperator; + airflow.operators.python.PythonOperator" default: "" version_added: 1.1.0 selective_enable: diff --git a/airflow/providers/standard/core/__init__.py b/airflow/providers/standard/core/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/standard/core/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/standard/core/operators/__init__.py b/airflow/providers/standard/core/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/standard/core/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/operators/bash.py b/airflow/providers/standard/core/operators/bash.py similarity index 100% rename from airflow/operators/bash.py rename to airflow/providers/standard/core/operators/bash.py diff --git a/airflow/providers/standard/core/sensors/__init__.py b/airflow/providers/standard/core/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/standard/core/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/sensors/bash.py b/airflow/providers/standard/core/sensors/bash.py similarity index 100% rename from airflow/sensors/bash.py rename to airflow/providers/standard/core/sensors/bash.py diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4b28c7a0a63d8..61b8f61a0609c 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -703,7 +703,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Only Always and common providers tests should run when only common.io and tests/always changed", ), pytest.param( - ("airflow/operators/bash.py",), + ("airflow/providers/standard/core/operators/bash.py",), { "affected-providers-list-as-string": None, "all-python-versions": "['3.9']", diff --git a/dev/perf/dags/elastic_dag.py b/dev/perf/dags/elastic_dag.py index e0adcdf5caf11..9805e71053883 100644 --- a/dev/perf/dags/elastic_dag.py +++ b/dev/perf/dags/elastic_dag.py @@ -24,7 +24,7 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator # DAG File used in performance tests. Its shape can be configured by environment variables. RE_TIME_DELTA = re.compile( diff --git a/dev/perf/dags/perf_dag_2.py b/dev/perf/dags/perf_dag_2.py index 641bb7565c8f3..bc87aa0b433d8 100644 --- a/dev/perf/dags/perf_dag_2.py +++ b/dev/perf/dags/perf_dag_2.py @@ -24,7 +24,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator args = { "owner": "airflow", diff --git a/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst b/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst index a52540fe78282..6e8c27990766b 100644 --- a/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.chime import send_chime_notification with DAG( diff --git a/docs/apache-airflow-providers-amazon/notifications/sns.rst b/docs/apache-airflow-providers-amazon/notifications/sns.rst index bbaad4f814712..d2b2b982750d3 100644 --- a/docs/apache-airflow-providers-amazon/notifications/sns.rst +++ b/docs/apache-airflow-providers-amazon/notifications/sns.rst @@ -33,7 +33,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.sns import send_sns_notification dag_failure_sns_notification = send_sns_notification( diff --git a/docs/apache-airflow-providers-amazon/notifications/sqs.rst b/docs/apache-airflow-providers-amazon/notifications/sqs.rst index 6951caa9fdd67..cdbd77856d530 100644 --- a/docs/apache-airflow-providers-amazon/notifications/sqs.rst +++ b/docs/apache-airflow-providers-amazon/notifications/sqs.rst @@ -33,7 +33,7 @@ Example Code: from datetime import datetime, timezone from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.sqs import send_sqs_notification dag_failure_sqs_notification = send_sqs_notification( diff --git a/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst b/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst index 777a3d46a3b2c..0d9c1769c36cb 100644 --- a/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst @@ -30,7 +30,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.apprise.notifications.apprise import send_apprise_notification from apprise import NotifyType diff --git a/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst b/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst index e0ed12558145f..e406a192a937d 100644 --- a/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst +++ b/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst @@ -31,7 +31,7 @@ Example Code from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.atlassian.jira.notifications.jira import send_jira_notification with DAG( diff --git a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst index 9c71e885d8d6a..91238fc61afe6 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst @@ -96,7 +96,7 @@ instead. You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` fields to dynamically determine their values. The result are saved to :ref:`XCom `, allowing them to be used by other operators. In this case, the -:class:`~airflow.operators.bash.BashOperator` is used to print the model information. +:class:`~airflow.providers.standard.core.operators.bash.BashOperator` is used to print the model information. .. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py :language: python diff --git a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst index 7d9a6dd5ff2ab..a5245ae563af3 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst @@ -101,7 +101,7 @@ Also for this action you can use sensor in the deferrable mode: :start-after: [START howto_operator_gcp_pubsub_pull_message_with_operator] :end-before: [END howto_operator_gcp_pubsub_pull_message_with_operator] -To pull messages from XCom use the :class:`~airflow.operators.bash.BashOperator`. +To pull messages from XCom use the :class:`~airflow.providers.standard.core.operators.bash.BashOperator`. .. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py :language: python diff --git a/docs/apache-airflow-providers-openlineage/guides/developer.rst b/docs/apache-airflow-providers-openlineage/guides/developer.rst index c2a5ffdc8fda4..ea179ddf507af 100644 --- a/docs/apache-airflow-providers-openlineage/guides/developer.rst +++ b/docs/apache-airflow-providers-openlineage/guides/developer.rst @@ -390,7 +390,7 @@ An Operator inside the Airflow DAG can be annotated with inlets and outlets like import pendulum from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.lineage.entities import Table, File, Column, User diff --git a/docs/apache-airflow-providers-openlineage/guides/user.rst b/docs/apache-airflow-providers-openlineage/guides/user.rst index 2c299b8c6d39e..9eb6ec69bae9a 100644 --- a/docs/apache-airflow-providers-openlineage/guides/user.rst +++ b/docs/apache-airflow-providers-openlineage/guides/user.rst @@ -257,13 +257,13 @@ full import paths of Airflow Operators to disable as ``disabled_for_operators`` [openlineage] transport = {"type": "http", "url": "http://example.com:5000", "endpoint": "api/v1/lineage"} - disabled_for_operators = 'airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator' + disabled_for_operators = 'airflow.providers.standard.core.operators.bash.BashOperator;airflow.operators.python.PythonOperator' ``AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS`` environment variable is an equivalent. .. code-block:: ini - AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator' + AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.providers.standard.core.operators.bash.BashOperator;airflow.operators.python.PythonOperator' Full Task Info ^^^^^^^^^^^^^^ diff --git a/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst b/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst index d16f9b2b9e48a..88db5ef6732b1 100644 --- a/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.pagerduty.notifications.pagerduty import send_pagerduty_notification with DAG( diff --git a/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst b/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst index a4f891f8a57bb..380e916c75f7d 100644 --- a/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.slack.notifications.slack import send_slack_notification with DAG( diff --git a/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst b/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst index 66ced818a7d18..bf404f77f2d33 100644 --- a/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst @@ -32,7 +32,7 @@ Example Code: from datetime import datetime, timezone from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.slack.notifications.slack_webhook import send_slack_webhook_notification dag_failure_slack_webhook_notification = send_slack_webhook_notification( diff --git a/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst b/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst index 4cb1bf310e03d..68a41c1294df8 100644 --- a/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.smtp.notifications.smtp import send_smtp_notification with DAG( diff --git a/docs/apache-airflow/administration-and-deployment/lineage.rst b/docs/apache-airflow/administration-and-deployment/lineage.rst index b274809175c03..b2c1664a3a54c 100644 --- a/docs/apache-airflow/administration-and-deployment/lineage.rst +++ b/docs/apache-airflow/administration-and-deployment/lineage.rst @@ -36,7 +36,7 @@ works. from airflow.lineage import AUTO from airflow.lineage.entities import File from airflow.models import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator FILE_CATEGORIES = ["CAT1", "CAT2", "CAT3"] diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst index 80a5996f36768..b11ba19aaced0 100644 --- a/docs/apache-airflow/best-practices.rst +++ b/docs/apache-airflow/best-practices.rst @@ -480,7 +480,7 @@ It's easier to grab the concept with an example. Let's say that we have the foll from airflow import DAG from airflow.decorators import task from airflow.exceptions import AirflowException - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule diff --git a/docs/apache-airflow/core-concepts/dag-run.rst b/docs/apache-airflow/core-concepts/dag-run.rst index 0621d3f771e2c..7f7d8bec0ed3a 100644 --- a/docs/apache-airflow/core-concepts/dag-run.rst +++ b/docs/apache-airflow/core-concepts/dag-run.rst @@ -101,7 +101,7 @@ in the configuration file. When turned off, the scheduler creates a DAG run only https://github.com/apache/airflow/blob/main/airflow/example_dags/tutorial.py """ from airflow.models.dag import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator import datetime import pendulum @@ -241,7 +241,7 @@ Example of a parameterized DAG: import pendulum from airflow import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator dag = DAG( "example_parameterized_dag", diff --git a/docs/apache-airflow/core-concepts/dags.rst b/docs/apache-airflow/core-concepts/dags.rst index f9dc7d64c72e0..b0bf1181d276d 100644 --- a/docs/apache-airflow/core-concepts/dags.rst +++ b/docs/apache-airflow/core-concepts/dags.rst @@ -574,7 +574,7 @@ TaskGroup also supports ``default_args`` like DAG, it will overwrite the ``defau from airflow import DAG from airflow.decorators import task_group - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator with DAG( diff --git a/docs/apache-airflow/core-concepts/operators.rst b/docs/apache-airflow/core-concepts/operators.rst index 354697c308537..df746ba3b99a6 100644 --- a/docs/apache-airflow/core-concepts/operators.rst +++ b/docs/apache-airflow/core-concepts/operators.rst @@ -28,7 +28,7 @@ An Operator is conceptually a template for a predefined :doc:`Task `, tha Airflow has a very extensive set of operators available, with some built-in to the core or pre-installed providers. Some popular operators from core include: -- :class:`~airflow.operators.bash.BashOperator` - executes a bash command +- :class:`~airflow.providers.standard.core.operators.bash.BashOperator` - executes a bash command - :class:`~airflow.operators.python.PythonOperator` - calls an arbitrary Python function - :class:`~airflow.operators.email.EmailOperator` - sends an email - Use the ``@task`` decorator to execute an arbitrary Python function. It doesn't support rendering jinja templates passed as arguments. diff --git a/docs/apache-airflow/core-concepts/tasks.rst b/docs/apache-airflow/core-concepts/tasks.rst index ad03283ef772d..92094e3bc4a85 100644 --- a/docs/apache-airflow/core-concepts/tasks.rst +++ b/docs/apache-airflow/core-concepts/tasks.rst @@ -236,7 +236,7 @@ If you'd like to reproduce zombie tasks for development/testing processes, follo .. code-block:: python from airflow.decorators import dag - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from datetime import datetime diff --git a/docs/apache-airflow/howto/notifications.rst b/docs/apache-airflow/howto/notifications.rst index c477ec1d3c173..2a8f547c486af 100644 --- a/docs/apache-airflow/howto/notifications.rst +++ b/docs/apache-airflow/howto/notifications.rst @@ -59,7 +59,7 @@ Here's an example of using the above notifier: from datetime import datetime from airflow.models.dag import DAG - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator from myprovider.notifier import MyNotifier diff --git a/docs/apache-airflow/howto/operator/bash.rst b/docs/apache-airflow/howto/operator/bash.rst index daf430fa14cde..21ca26d53b0d3 100644 --- a/docs/apache-airflow/howto/operator/bash.rst +++ b/docs/apache-airflow/howto/operator/bash.rst @@ -22,7 +22,7 @@ BashOperator ============ -Use the :class:`~airflow.operators.bash.BashOperator` to execute +Use the :class:`~airflow.providers.standard.core.operators.bash.BashOperator` to execute commands in a `Bash `__ shell. The Bash command or script to execute is determined by: @@ -390,7 +390,7 @@ There are numerous possibilities with this type of pre-execution enrichment. BashSensor ========== -Use the :class:`~airflow.sensors.bash.BashSensor` to use arbitrary command for sensing. The command +Use the :class:`~airflow.providers.standard.core.sensors.bash.BashSensor` to use arbitrary command for sensing. The command should return 0 when it succeeds, any other value otherwise. .. exampleinclude:: /../../airflow/example_dags/example_sensors.py diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst index 44dcd9a3bd36c..e4b1dd6ddc05c 100644 --- a/docs/apache-airflow/index.rst +++ b/docs/apache-airflow/index.rst @@ -41,7 +41,7 @@ Take a look at the following snippet of code: from airflow import DAG from airflow.decorators import task - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator # A DAG represents a workflow, a collection of tasks with DAG(dag_id="demo", start_date=datetime(2022, 1, 1), schedule="0 0 * * *") as dag: diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst index 16b74305a958b..fe12ac2f5d12a 100644 --- a/docs/apache-airflow/operators-and-hooks-ref.rst +++ b/docs/apache-airflow/operators-and-hooks-ref.rst @@ -50,7 +50,7 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Operators - Guides - * - :mod:`airflow.operators.bash` + * - :mod:`airflow.providers.standard.core.operators.bash` - :doc:`How to use ` * - :mod:`airflow.operators.branch` @@ -82,7 +82,7 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Sensors - Guides - * - :mod:`airflow.sensors.bash` + * - :mod:`airflow.providers.standard.core.sensors.bash` - :ref:`How to use ` * - :mod:`airflow.sensors.external_task` diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index c77debab8f328..a8402a26ea0af 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -437,7 +437,7 @@ the parameter value is used. Adding dependencies between decorated and traditional tasks ----------------------------------------------------------- The above tutorial shows how to create dependencies between TaskFlow functions. However, dependencies can also -be set between traditional tasks (such as :class:`~airflow.operators.bash.BashOperator` +be set between traditional tasks (such as :class:`~airflow.providers.standard.core.operators.bash.BashOperator` or :class:`~airflow.sensors.filesystem.FileSensor`) and TaskFlow functions. Building this dependency is shown in the code below: diff --git a/docs/exts/templates/openlineage.rst.jinja2 b/docs/exts/templates/openlineage.rst.jinja2 index 7dffc175f84b2..1fc106bebbad8 100644 --- a/docs/exts/templates/openlineage.rst.jinja2 +++ b/docs/exts/templates/openlineage.rst.jinja2 @@ -22,7 +22,7 @@ At the moment, two core operators supports OpenLineage. These operators function capable of running any code, which might limit the extent of lineage extraction. - :class:`~airflow.operators.python.PythonOperator` (via :class:`airflow.providers.openlineage.extractors.python.PythonExtractor`) -- :class:`~airflow.operators.bash.BashOperator` (via :class:`airflow.providers.openlineage.extractors.bash.BashExtractor`) +- :class:`~airflow.providers.standard.core.operators.bash.BashOperator` (via :class:`airflow.providers.openlineage.extractors.bash.BashExtractor`) :class:`~airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator` diff --git a/tests/callbacks/test_callback_requests.py b/tests/callbacks/test_callback_requests.py index 5992ee6fbbf70..f995764a9e6f6 100644 --- a/tests/callbacks/test_callback_requests.py +++ b/tests/callbacks/test_callback_requests.py @@ -27,7 +27,7 @@ ) from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType @@ -100,7 +100,7 @@ def test_simple_ti_roundtrip_exec_config_pod(self): from airflow.callbacks.callback_requests import TaskCallbackRequest from airflow.models import TaskInstance from airflow.models.taskinstance import SimpleTaskInstance - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator test_pod = k8s.V1Pod(metadata=k8s.V1ObjectMeta(name="hello", namespace="ns")) op = BashOperator(task_id="hi", executor_config={"pod_override": test_pod}, bash_command="hi") @@ -115,7 +115,7 @@ def test_simple_ti_roundtrip_dates(self, dag_maker): from airflow.callbacks.callback_requests import TaskCallbackRequest from airflow.models import TaskInstance from airflow.models.taskinstance import SimpleTaskInstance - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator with dag_maker(schedule=timedelta(weeks=1), serialized=True): op = BashOperator(task_id="hi", bash_command="hi") diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index 36cbcc85a7214..ef0b470551585 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -45,8 +45,8 @@ from airflow.executors.local_executor import LocalExecutor from airflow.models import DagBag, DagRun, Pool, TaskInstance from airflow.models.serialized_dag import SerializedDagModel -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState diff --git a/tests/core/test_core.py b/tests/core/test_core.py index a75428b33a18b..eb7702370af5a 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -27,9 +27,9 @@ from airflow.exceptions import AirflowTaskTimeout from airflow.models import DagRun, TaskFail, TaskInstance from airflow.models.baseoperator import BaseOperator -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail diff --git a/tests/dags/subdir2/test_dont_ignore_this.py b/tests/dags/subdir2/test_dont_ignore_this.py index 72c1796a424b2..cc25f47587764 100644 --- a/tests/dags/subdir2/test_dont_ignore_this.py +++ b/tests/dags/subdir2/test_dont_ignore_this.py @@ -20,7 +20,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator DEFAULT_DATE = datetime(2019, 12, 1) diff --git a/tests/dags/test_assets.py b/tests/dags/test_assets.py index a4ecd6aad4a6a..7d82c27bb42a0 100644 --- a/tests/dags/test_assets.py +++ b/tests/dags/test_assets.py @@ -22,8 +22,8 @@ from airflow.assets import Asset from airflow.exceptions import AirflowFailException, AirflowSkipException from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator skip_task_dag_dataset = Asset("s3://dag_with_skip_task/output_1.txt", extra={"hi": "bye"}) fail_task_dag_dataset = Asset("s3://dag_with_fail_task/output_1.txt", extra={"hi": "bye"}) diff --git a/tests/dags/test_backfill_with_upstream_failed_task.py b/tests/dags/test_backfill_with_upstream_failed_task.py index d2cb6353bfaa3..90c1ec8ee93bf 100644 --- a/tests/dags/test_backfill_with_upstream_failed_task.py +++ b/tests/dags/test_backfill_with_upstream_failed_task.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator dag = DAG( dag_id="test_backfill_with_upstream_failed_task", diff --git a/tests/dags/test_default_impersonation.py b/tests/dags/test_default_impersonation.py index 468b7dce072dd..dc3f45c16e078 100644 --- a/tests/dags/test_default_impersonation.py +++ b/tests/dags/test_default_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_example_bash_operator.py b/tests/dags/test_example_bash_operator.py index eb472b8011ebd..55739af8ca3bb 100644 --- a/tests/dags/test_example_bash_operator.py +++ b/tests/dags/test_example_bash_operator.py @@ -20,8 +20,8 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.core.operators.bash import BashOperator dag = DAG( dag_id="test_example_bash_operator", diff --git a/tests/dags/test_failing.py b/tests/dags/test_failing.py index 28e2fb5881987..58c681dbdbb56 100644 --- a/tests/dags/test_failing.py +++ b/tests/dags/test_failing.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator dag = DAG( dag_id="test_failing_bash_operator", diff --git a/tests/dags/test_heartbeat_failed_fast.py b/tests/dags/test_heartbeat_failed_fast.py index aee7a67030585..631e37bcf86ef 100644 --- a/tests/dags/test_heartbeat_failed_fast.py +++ b/tests/dags/test_heartbeat_failed_fast.py @@ -20,7 +20,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_impersonation.py b/tests/dags/test_impersonation.py index 33a3c89d328d9..d27d826f88dfc 100644 --- a/tests/dags/test_impersonation.py +++ b/tests/dags/test_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_miscellaneous.py b/tests/dags/test_miscellaneous.py index c19277a617571..ccdffa0dd2275 100644 --- a/tests/dags/test_miscellaneous.py +++ b/tests/dags/test_miscellaneous.py @@ -22,8 +22,8 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.core.operators.bash import BashOperator args = { "owner": "airflow", diff --git a/tests/dags/test_multiple_dags.py b/tests/dags/test_multiple_dags.py index 5801084fab712..d489bc8468553 100644 --- a/tests/dags/test_multiple_dags.py +++ b/tests/dags/test_multiple_dags.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator args = {"owner": "airflow", "retries": 3, "start_date": datetime.datetime(2022, 1, 1)} diff --git a/tests/dags/test_no_impersonation.py b/tests/dags/test_no_impersonation.py index 2a75d5321473c..8d320866a882c 100644 --- a/tests/dags/test_no_impersonation.py +++ b/tests/dags/test_no_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_on_failure_callback.py b/tests/dags/test_on_failure_callback.py index e2f4ab9027a8c..0d130e60a06d7 100644 --- a/tests/dags/test_on_failure_callback.py +++ b/tests/dags/test_on_failure_callback.py @@ -21,8 +21,8 @@ from airflow.exceptions import AirflowFailException from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_retry_handling_job.py b/tests/dags/test_retry_handling_job.py index 7040e8c8756f8..e17a084a28503 100644 --- a/tests/dags/test_retry_handling_job.py +++ b/tests/dags/test_retry_handling_job.py @@ -20,7 +20,7 @@ from datetime import datetime, timedelta from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator default_args = { "owner": "airflow", diff --git a/tests/decorators/test_setup_teardown.py b/tests/decorators/test_setup_teardown.py index 13451ba379eca..13aaed23dc6a1 100644 --- a/tests/decorators/test_setup_teardown.py +++ b/tests/decorators/test_setup_teardown.py @@ -22,7 +22,7 @@ from airflow.decorators import setup, task, task_group, teardown from airflow.decorators.setup_teardown import context_wrapper from airflow.exceptions import AirflowException -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 9c7fe96ff186d..8df14eb318eb4 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -43,7 +43,7 @@ from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance from airflow.models.taskinstancekey import TaskInstanceKey -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.state import State, TaskInstanceState from tests.test_utils import db diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index b4e4c10cff456..7c5b106bae685 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -60,8 +60,8 @@ from airflow.models.pool import Pool from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance, TaskInstanceKey -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone from airflow.utils.file import list_py_file_paths diff --git a/tests/listeners/test_listeners.py b/tests/listeners/test_listeners.py index 3c34ab0ff8ab2..9edafd0992ee3 100644 --- a/tests/listeners/test_listeners.py +++ b/tests/listeners/test_listeners.py @@ -25,7 +25,7 @@ from airflow.exceptions import AirflowException from airflow.jobs.job import Job, run_job from airflow.listeners.listener import get_listener_manager -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import DagRunState, TaskInstanceState diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index ab67c3778c262..4d84198596d06 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -74,9 +74,9 @@ from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskfail import TaskFail from airflow.models.taskinstance import TaskInstance as TI -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.security import permissions from airflow.templates import NativeEnvironment, SandboxedEnvironment from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index c7dacaeb291e4..9bf50b58412fe 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -36,9 +36,9 @@ from airflow.models.taskinstance import TaskInstance, TaskInstanceNote, clear_task_instances from airflow.models.taskmap import TaskMap from airflow.models.taskreschedule import TaskReschedule -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import ShortCircuitOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.stats import Stats from airflow.triggers.base import StartTriggerArgs diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index b8c45193814aa..390c686942e70 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -31,7 +31,7 @@ from airflow.decorators import task as task_decorator from airflow.models import Variable from airflow.models.renderedtifields import RenderedTaskInstanceFields as RTIF -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.timezone import datetime from tests.test_utils.asserts import assert_queries_count diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py index d9a77e55edaf5..7a64d4e4cc1a7 100644 --- a/tests/models/test_serialized_dag.py +++ b/tests/models/test_serialized_dag.py @@ -31,7 +31,7 @@ from airflow.models.dagbag import DagBag from airflow.models.dagcode import DagCode from airflow.models.serialized_dag import SerializedDagModel as SDM -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 8c334366f0488..261c4279c0d3c 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -76,9 +76,9 @@ from airflow.models.variable import Variable from airflow.models.xcom import LazyXComSelectSequence, XCom from airflow.notifications.basenotifier import BaseNotifier -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.sensors.base import BaseSensorOperator from airflow.sensors.python import PythonSensor from airflow.serialization.serialized_objects import SerializedBaseOperator, SerializedDAG diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index 6108c5e81930f..3e0062e9d24a9 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -19,8 +19,8 @@ import pytest from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.types import NOTSET from tests.test_utils.config import conf_vars from tests.test_utils.db import clear_db_dags, clear_db_runs diff --git a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py index 4622d31b575fc..46ddd61276fe8 100644 --- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -30,7 +30,6 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models.taskinstancekey import TaskInstanceKey -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.cncf.kubernetes import pod_generator from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import ( @@ -53,6 +52,7 @@ get_logs_task_metadata, ) from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State, TaskInstanceState from tests.test_utils.config import conf_vars diff --git a/tests/providers/cncf/kubernetes/test_template_rendering.py b/tests/providers/cncf/kubernetes/test_template_rendering.py index ab2820284d553..2b4644bf6dc06 100644 --- a/tests/providers/cncf/kubernetes/test_template_rendering.py +++ b/tests/providers/cncf/kubernetes/test_template_rendering.py @@ -24,8 +24,8 @@ from airflow.configuration import TEST_DAGS_FOLDER from airflow.models.renderedtifields import RenderedTaskInstanceFields, RenderedTaskInstanceFields as RTIF -from airflow.operators.bash import BashOperator from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec, render_k8s_pod_yaml +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.session import create_session from airflow.version import version from tests.models import DEFAULT_DATE diff --git a/tests/providers/openlineage/extractors/test_bash.py b/tests/providers/openlineage/extractors/test_bash.py index de65a1d176d8a..e33766e57af24 100644 --- a/tests/providers/openlineage/extractors/test_bash.py +++ b/tests/providers/openlineage/extractors/test_bash.py @@ -26,8 +26,8 @@ from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.operators.bash import BashOperator from airflow.providers.openlineage.extractors.bash import BashExtractor +from airflow.providers.standard.core.operators.bash import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/extractors/test_python.py b/tests/providers/openlineage/extractors/test_python.py index 81284383d8648..a6e73a9c5997d 100644 --- a/tests/providers/openlineage/extractors/test_python.py +++ b/tests/providers/openlineage/extractors/test_python.py @@ -28,9 +28,9 @@ from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from airflow.providers.openlineage.extractors.python import PythonExtractor +from airflow.providers.standard.core.operators.bash import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/plugins/test_adapter.py b/tests/providers/openlineage/plugins/test_adapter.py index 260883470875f..8c1a758371605 100644 --- a/tests/providers/openlineage/plugins/test_adapter.py +++ b/tests/providers/openlineage/plugins/test_adapter.py @@ -40,7 +40,6 @@ from airflow import DAG from airflow.models.dagrun import DagRun, DagRunState from airflow.models.taskinstance import TaskInstance, TaskInstanceState -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.openlineage.conf import namespace from airflow.providers.openlineage.extractors import OperatorLineage @@ -51,6 +50,7 @@ AirflowStateRunFacet, ) from airflow.providers.openlineage.utils.utils import get_airflow_job_facet +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup from tests.test_utils.config import conf_vars diff --git a/tests/providers/openlineage/plugins/test_facets.py b/tests/providers/openlineage/plugins/test_facets.py index 0ed5b4bf7c440..5a33b27ced243 100644 --- a/tests/providers/openlineage/plugins/test_facets.py +++ b/tests/providers/openlineage/plugins/test_facets.py @@ -81,7 +81,7 @@ def test_airflow_dag_run_facet(): }, tasks={ "task_0": { - "operator": "airflow.operators.bash.BashOperator", + "operator": "airflow.providers.standard.core.operators.bash.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": "#f0ede4", diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py index 5335739a8ff92..61eb2b1b0721a 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/tests/providers/openlineage/plugins/test_utils.py @@ -29,7 +29,6 @@ from pkg_resources import parse_version from airflow.models import DAG as AIRFLOW_DAG, DagModel -from airflow.operators.bash import BashOperator from airflow.providers.openlineage.plugins.facets import AirflowDebugRunFacet from airflow.providers.openlineage.utils.utils import ( InfoJsonEncodable, @@ -41,6 +40,7 @@ get_fully_qualified_class_name, is_operator_disabled, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.log.secrets_masker import _secrets_masker from airflow.utils.state import State @@ -262,7 +262,7 @@ def test_get_fully_qualified_class_name(): from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter result = get_fully_qualified_class_name(BashOperator(task_id="test", bash_command="exit 0;")) - assert result == "airflow.operators.bash.BashOperator" + assert result == "airflow.providers.standard.core.operators.bash.BashOperator" result = get_fully_qualified_class_name(OpenLineageAdapter()) assert result == "airflow.providers.openlineage.plugins.adapter.OpenLineageAdapter" @@ -278,7 +278,7 @@ def test_is_operator_disabled(mock_disabled_operators): assert is_operator_disabled(op) is False mock_disabled_operators.return_value = { - "airflow.operators.bash.BashOperator", + "airflow.providers.standard.core.operators.bash.BashOperator", "airflow.operators.python.PythonOperator", } assert is_operator_disabled(op) is True @@ -303,7 +303,7 @@ def test_includes_full_task_info(mock_include_full_task_info): @patch("airflow.providers.openlineage.conf.include_full_task_info") def test_does_not_include_full_task_info(mock_include_full_task_info): - from airflow.operators.bash import BashOperator + from airflow.providers.standard.core.operators.bash import BashOperator mock_include_full_task_info.return_value = False # There should be no 'bash_command' in excludes and it's not in includes - so diff --git a/tests/providers/openlineage/utils/test_utils.py b/tests/providers/openlineage/utils/test_utils.py index d97a447e99949..fdad1f41966e2 100644 --- a/tests/providers/openlineage/utils/test_utils.py +++ b/tests/providers/openlineage/utils/test_utils.py @@ -27,7 +27,6 @@ from airflow.models.dagrun import DagRun from airflow.models.mappedoperator import MappedOperator from airflow.models.taskinstance import TaskInstance, TaskInstanceState -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator from airflow.providers.openlineage.plugins.facets import AirflowDagRunFacet, AirflowJobFacet @@ -41,6 +40,7 @@ get_operator_class, get_user_provided_run_facets, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedBaseOperator from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType @@ -82,7 +82,7 @@ def test_get_airflow_job_facet(): }, tasks={ "task_0": { - "operator": "airflow.operators.bash.BashOperator", + "operator": "airflow.providers.standard.core.operators.bash.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": "#f0ede4", @@ -166,7 +166,7 @@ def test_get_airflow_dag_run_facet(): def test_get_fully_qualified_class_name_serialized_operator(): - op_module_path = "airflow.operators.bash" + op_module_path = "airflow.providers.standard.core.operators.bash" op_name = "BashOperator" op = BashOperator(task_id="test", bash_command="echo 1") @@ -191,7 +191,7 @@ def test_get_fully_qualified_class_name_mapped_operator(): def test_get_fully_qualified_class_name_bash_operator(): result = get_fully_qualified_class_name(BashOperator(task_id="test", bash_command="echo 0;")) - expected_result = "airflow.operators.bash.BashOperator" + expected_result = "airflow.providers.standard.core.operators.bash.BashOperator" assert result == expected_result @@ -319,7 +319,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_0": { - "operator": "airflow.operators.bash.BashOperator", + "operator": "airflow.providers.standard.core.operators.bash.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -360,7 +360,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_3": { - "operator": "airflow.operators.bash.BashOperator", + "operator": "airflow.providers.standard.core.operators.bash.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -388,7 +388,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_5": { - "operator": "airflow.operators.bash.BashOperator", + "operator": "airflow.providers.standard.core.operators.bash.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, diff --git a/tests/providers/standard/core/__init__.py b/tests/providers/standard/core/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/standard/core/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/standard/core/operators/__init__.py b/tests/providers/standard/core/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/standard/core/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/operators/test_bash.py b/tests/providers/standard/core/operators/test_bash.py similarity index 99% rename from tests/operators/test_bash.py rename to tests/providers/standard/core/operators/test_bash.py index 8aacb3b7c77ea..6b579c044daf4 100644 --- a/tests/operators/test_bash.py +++ b/tests/providers/standard/core/operators/test_bash.py @@ -28,7 +28,7 @@ import pytest from airflow.exceptions import AirflowException, AirflowSkipException, AirflowTaskTimeout -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType diff --git a/tests/providers/standard/core/sensors/__init__.py b/tests/providers/standard/core/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/standard/core/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/sensors/test_bash.py b/tests/providers/standard/core/sensors/test_bash.py similarity index 97% rename from tests/sensors/test_bash.py rename to tests/providers/standard/core/sensors/test_bash.py index 3282f6b971221..212066498df1c 100644 --- a/tests/sensors/test_bash.py +++ b/tests/providers/standard/core/sensors/test_bash.py @@ -23,7 +23,7 @@ from airflow.exceptions import AirflowFailException, AirflowSensorTimeout from airflow.models.dag import DAG -from airflow.sensors.bash import BashSensor +from airflow.providers.standard.core.sensors.bash import BashSensor class TestBashSensor: diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index d063b3e78035a..4d5497bd194a4 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -60,11 +60,11 @@ from airflow.models.mappedoperator import MappedOperator from airflow.models.param import Param, ParamsDict from airflow.models.xcom import XCom -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator +from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.core.sensors.bash import BashSensor from airflow.security import permissions -from airflow.sensors.bash import BashSensor from airflow.serialization.enums import Encoding from airflow.serialization.json_schema import load_dag_schema_dict from airflow.serialization.serialized_objects import ( @@ -154,7 +154,7 @@ "template_fields_renderers": {"bash_command": "bash", "env": "json"}, "bash_command": "echo {{ task.task_id }}", "_task_type": "BashOperator", - "_task_module": "airflow.operators.bash", + "_task_module": "airflow.providers.standard.core.operators.bash", "pool": "default_pool", "is_setup": False, "is_teardown": False, @@ -2284,7 +2284,7 @@ def test_operator_expand_serde(): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "airflow.operators.bash", + "_task_module": "airflow.providers.standard.core.operators.bash", "_task_type": "BashOperator", "start_trigger_args": None, "start_from_trigger": False, diff --git a/tests/system/core/example_external_task_child_deferrable.py b/tests/system/core/example_external_task_child_deferrable.py index 9af83b7699ab1..b6feac48d14d2 100644 --- a/tests/system/core/example_external_task_child_deferrable.py +++ b/tests/system/core/example_external_task_child_deferrable.py @@ -19,7 +19,7 @@ from datetime import datetime from airflow import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator with DAG( dag_id="child_dag", diff --git a/tests/system/providers/amazon/aws/example_appflow.py b/tests/system/providers/amazon/aws/example_appflow.py index 0fb2764c0b71c..60c18f36be5ea 100644 --- a/tests/system/providers/amazon/aws/example_appflow.py +++ b/tests/system/providers/amazon/aws/example_appflow.py @@ -20,7 +20,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.amazon.aws.operators.appflow import ( AppflowRecordsShortCircuitOperator, AppflowRunAfterOperator, @@ -28,6 +27,7 @@ AppflowRunDailyOperator, AppflowRunFullOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() diff --git a/tests/system/providers/amazon/aws/example_http_to_s3.py b/tests/system/providers/amazon/aws/example_http_to_s3.py index 3654140b4a1ed..0e6ad0daee0b0 100644 --- a/tests/system/providers/amazon/aws/example_http_to_s3.py +++ b/tests/system/providers/amazon/aws/example_http_to_s3.py @@ -23,9 +23,9 @@ from airflow.models.baseoperator import chain from airflow.models.connection import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.http_to_s3 import HttpToS3Operator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder diff --git a/tests/system/providers/amazon/aws/utils/k8s.py b/tests/system/providers/amazon/aws/utils/k8s.py index 551d09629e9d3..a27e5eeb27275 100644 --- a/tests/system/providers/amazon/aws/utils/k8s.py +++ b/tests/system/providers/amazon/aws/utils/k8s.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator if TYPE_CHECKING: from airflow.models.operator import Operator diff --git a/tests/system/providers/apache/hive/example_twitter_dag.py b/tests/system/providers/apache/hive/example_twitter_dag.py index 53b824f50923e..fdcd6973402ef 100644 --- a/tests/system/providers/apache/hive/example_twitter_dag.py +++ b/tests/system/providers/apache/hive/example_twitter_dag.py @@ -26,8 +26,8 @@ from airflow import DAG from airflow.decorators import task -from airflow.operators.bash import BashOperator from airflow.providers.apache.hive.operators.hive import HiveOperator +from airflow.providers.standard.core.operators.bash import BashOperator # -------------------------------------------------------------------------------- # Caveat: This Dag will not run because of missing scripts. diff --git a/tests/system/providers/apache/iceberg/example_iceberg.py b/tests/system/providers/apache/iceberg/example_iceberg.py index 0318a8e22b770..4f144a6344bb8 100644 --- a/tests/system/providers/apache/iceberg/example_iceberg.py +++ b/tests/system/providers/apache/iceberg/example_iceberg.py @@ -19,8 +19,8 @@ from datetime import datetime, timedelta from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.apache.iceberg.hooks.iceberg import IcebergHook +from airflow.providers.standard.core.operators.bash import BashOperator bash_command = f""" echo "Our token: {IcebergHook().get_token_macro()}" diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes.py b/tests/system/providers/cncf/kubernetes/example_kubernetes.py index 57bab063a9e62..4220687448633 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes.py @@ -27,9 +27,9 @@ from kubernetes.client import models as k8s from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret +from airflow.providers.standard.core.operators.bash import BashOperator # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py index 881bfd61f7c8d..9356fdb9ba269 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py @@ -27,9 +27,9 @@ from kubernetes.client import models as k8s from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret +from airflow.providers.standard.core.operators.bash import BashOperator # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/tests/system/providers/docker/example_docker.py b/tests/system/providers/docker/example_docker.py index 069f4794de632..009b137553edd 100644 --- a/tests/system/providers/docker/example_docker.py +++ b/tests/system/providers/docker/example_docker.py @@ -21,8 +21,8 @@ from datetime import datetime from airflow import models -from airflow.operators.bash import BashOperator from airflow.providers.docker.operators.docker import DockerOperator +from airflow.providers.standard.core.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "docker_test" diff --git a/tests/system/providers/docker/example_docker_copy_data.py b/tests/system/providers/docker/example_docker_copy_data.py index 50373af792515..0086bdf59c65d 100644 --- a/tests/system/providers/docker/example_docker_copy_data.py +++ b/tests/system/providers/docker/example_docker_copy_data.py @@ -32,9 +32,9 @@ from docker.types import Mount from airflow import models -from airflow.operators.bash import BashOperator from airflow.operators.python import ShortCircuitOperator from airflow.providers.docker.operators.docker import DockerOperator +from airflow.providers.standard.core.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "docker_sample_copy_data" diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py index bd74e49d40964..246e673146e7b 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py @@ -25,13 +25,13 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.bigquery import ( BigQueryCreateEmptyDatasetOperator, BigQueryDeleteDatasetOperator, BigQueryGetDatasetOperator, BigQueryUpdateDatasetOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py index 6878a9822de1e..436db58b5bf43 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py @@ -25,7 +25,6 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.bigquery import ( BigQueryCheckOperator, BigQueryColumnCheckOperator, @@ -38,6 +37,7 @@ BigQueryTableCheckOperator, BigQueryValueCheckOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py index 2f1ed573aa053..410f4debcb3bc 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py @@ -25,7 +25,6 @@ from datetime import datetime, timedelta from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.bigquery import ( BigQueryCheckOperator, BigQueryCreateEmptyDatasetOperator, @@ -36,6 +35,7 @@ BigQueryIntervalCheckOperator, BigQueryValueCheckOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py index 0f7acd8a14d36..bb4b039ffae4c 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py @@ -36,7 +36,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -51,6 +50,7 @@ ComputeEngineInsertInstanceOperator, ) from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py index 990820bfe28c6..541f988d3370b 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py @@ -35,7 +35,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -50,6 +49,7 @@ ) from airflow.providers.google.cloud.transfers.bigquery_to_postgres import BigQueryToPostgresOperator from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py b/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py index 42cfbc8808f81..ac305c6d10c99 100644 --- a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +++ b/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py @@ -31,7 +31,6 @@ from airflow.decorators import task_group from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.cloud_build import ( CloudBuildCancelBuildOperator, CloudBuildCreateBuildOperator, @@ -39,6 +38,7 @@ CloudBuildListBuildsOperator, CloudBuildRetryBuildOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py index dce93182090d0..554d06253e0f1 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +++ b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py @@ -32,7 +32,6 @@ from google.protobuf.field_mask_pb2 import FieldMask from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.cloud_memorystore import ( CloudMemorystoreMemcachedApplyParametersOperator, CloudMemorystoreMemcachedCreateInstanceOperator, @@ -42,6 +41,7 @@ CloudMemorystoreMemcachedUpdateInstanceOperator, CloudMemorystoreMemcachedUpdateParametersOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py index aeee437ef9b57..971285ec64907 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +++ b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py @@ -27,7 +27,6 @@ from google.cloud.redis_v1 import FailoverInstanceRequest, Instance from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.cloud_memorystore import ( CloudMemorystoreCreateInstanceAndImportOperator, CloudMemorystoreCreateInstanceOperator, @@ -46,6 +45,7 @@ GCSCreateBucketOperator, GCSDeleteBucketOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py b/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py index 7d9046088a740..ceaf4ed82861f 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py +++ b/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py @@ -27,7 +27,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import ( GCSCreateBucketOperator, GCSDeleteBucketOperator, @@ -35,6 +34,7 @@ GCSListObjectsOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py index 9e92102fa924c..d1e12e3ee9988 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py @@ -29,7 +29,6 @@ from airflow.decorators import task from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from airflow.providers.google.cloud.operators.gcs import ( GCSCreateBucketOperator, @@ -39,6 +38,7 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py index 56fb0811b0621..7c776edd7f446 100644 --- a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py @@ -34,7 +34,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -47,6 +46,7 @@ GCSDeleteBucketOperator, ) from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py index 31951cd023010..21b997f821704 100644 --- a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py @@ -27,9 +27,9 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.sftp_to_gcs import SFTPToGCSOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_sheets.py b/tests/system/providers/google/cloud/gcs/example_sheets.py index 958bd90360285..bd831da4c4157 100644 --- a/tests/system/providers/google/cloud/gcs/example_sheets.py +++ b/tests/system/providers/google/cloud/gcs/example_sheets.py @@ -26,11 +26,11 @@ from airflow.models import Connection from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.sheets_to_gcs import GoogleSheetsToGCSOperator from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator from airflow.providers.google.suite.transfers.gcs_to_sheets import GCSToGoogleSheetsOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py index 031f8326ee99c..b58fb9ae60937 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py @@ -25,12 +25,12 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.kubernetes_engine import ( GKECreateClusterOperator, GKEDeleteClusterOperator, GKEStartPodOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py index 5e3f4ddbf7044..5462700745421 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py @@ -25,12 +25,12 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.kubernetes_engine import ( GKECreateClusterOperator, GKEDeleteClusterOperator, GKEStartPodOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py index 87602da88c46c..48af97834359f 100644 --- a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py +++ b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py @@ -29,7 +29,6 @@ from google.protobuf.struct_pb2 import Value from airflow import models -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import ( GCSCreateBucketOperator, GCSDeleteBucketOperator, @@ -53,6 +52,7 @@ ListModelVersionsOperator, SetDefaultVersionOnModelOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") diff --git a/tests/system/providers/google/cloud/natural_language/example_natural_language.py b/tests/system/providers/google/cloud/natural_language/example_natural_language.py index 5bc38f9220874..aecdba4f7ef68 100644 --- a/tests/system/providers/google/cloud/natural_language/example_natural_language.py +++ b/tests/system/providers/google/cloud/natural_language/example_natural_language.py @@ -27,13 +27,13 @@ from google.cloud.language_v1 import Document from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.natural_language import ( CloudNaturalLanguageAnalyzeEntitiesOperator, CloudNaturalLanguageAnalyzeEntitySentimentOperator, CloudNaturalLanguageAnalyzeSentimentOperator, CloudNaturalLanguageClassifyTextOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "gcp_natural_language" diff --git a/tests/system/providers/google/cloud/pubsub/example_pubsub.py b/tests/system/providers/google/cloud/pubsub/example_pubsub.py index 29ba6469ea0ce..157aef2e4fbfa 100644 --- a/tests/system/providers/google/cloud/pubsub/example_pubsub.py +++ b/tests/system/providers/google/cloud/pubsub/example_pubsub.py @@ -25,7 +25,6 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.pubsub import ( PubSubCreateSubscriptionOperator, PubSubCreateTopicOperator, @@ -35,6 +34,7 @@ PubSubPullOperator, ) from airflow.providers.google.cloud.sensors.pubsub import PubSubPullSensor +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py index cf256c8f9fa2f..58d98cc5c0780 100644 --- a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py +++ b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py @@ -34,7 +34,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -45,6 +44,7 @@ from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator from airflow.providers.google.suite.transfers.sql_to_sheets import SQLToGoogleSheetsOperator from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.settings import Session, json from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/tasks/example_queue.py b/tests/system/providers/google/cloud/tasks/example_queue.py index 53919fb146da6..f7e6436d91f9a 100644 --- a/tests/system/providers/google/cloud/tasks/example_queue.py +++ b/tests/system/providers/google/cloud/tasks/example_queue.py @@ -35,7 +35,6 @@ from airflow.decorators import task from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.tasks import ( CloudTasksQueueCreateOperator, CloudTasksQueueDeleteOperator, @@ -46,6 +45,7 @@ CloudTasksQueuesListOperator, CloudTasksQueueUpdateOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py index 8394c99fcef51..5f8086d3dc626 100644 --- a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py +++ b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py @@ -33,7 +33,6 @@ from airflow.decorators import task from airflow.models import Connection from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook from airflow.providers.google.cloud.hooks.compute_ssh import ComputeEngineSSHHook @@ -47,6 +46,7 @@ ) from airflow.providers.google.cloud.transfers.postgres_to_gcs import PostgresToGCSOperator from airflow.providers.ssh.operators.ssh import SSHOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/translate/example_translate.py b/tests/system/providers/google/cloud/translate/example_translate.py index 87f424673ecff..27b653cbf8359 100644 --- a/tests/system/providers/google/cloud/translate/example_translate.py +++ b/tests/system/providers/google/cloud/translate/example_translate.py @@ -25,8 +25,8 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.translate import CloudTranslateTextOperator +from airflow.providers.standard.core.operators.bash import BashOperator DAG_ID = "gcp_translate" diff --git a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py index eae6a54a89c3c..f56b1f47beb29 100644 --- a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +++ b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py @@ -33,7 +33,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.operators.video_intelligence import ( CloudVideoIntelligenceDetectVideoExplicitContentOperator, @@ -41,6 +40,7 @@ CloudVideoIntelligenceDetectVideoShotsOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py b/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py index 1d6167c6866ee..fb356e8c254fa 100644 --- a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +++ b/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py @@ -22,7 +22,6 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.operators.vision import ( CloudVisionDetectImageLabelsOperator, @@ -32,6 +31,7 @@ CloudVisionTextDetectOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule # [START howto_operator_vision_retry_import] diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py b/tests/system/providers/google/datacatalog/example_datacatalog_entries.py index db7d74b18d985..7947994836d3d 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_entries.py @@ -24,7 +24,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateEntryGroupOperator, CloudDataCatalogCreateEntryOperator, @@ -36,6 +35,7 @@ CloudDataCatalogUpdateEntryOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py b/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py index 8061ecaf110ae..96951f0b78a38 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py @@ -25,7 +25,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateEntryGroupOperator, CloudDataCatalogCreateEntryOperator, @@ -38,6 +37,7 @@ CloudDataCatalogSearchCatalogOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py b/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py index 6c1fa6f0cab45..69d91e39ee076 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py @@ -24,7 +24,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateTagTemplateFieldOperator, CloudDataCatalogCreateTagTemplateOperator, @@ -35,6 +34,7 @@ CloudDataCatalogUpdateTagTemplateFieldOperator, CloudDataCatalogUpdateTagTemplateOperator, ) +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py b/tests/system/providers/google/datacatalog/example_datacatalog_tags.py index 28764c3e0336d..cc841ddf67d92 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_tags.py @@ -25,7 +25,6 @@ from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.providers.google.cloud.operators.datacatalog import ( CloudDataCatalogCreateEntryGroupOperator, CloudDataCatalogCreateEntryOperator, @@ -39,6 +38,7 @@ CloudDataCatalogUpdateTagOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/opsgenie/example_opsgenie_notifier.py b/tests/system/providers/opsgenie/example_opsgenie_notifier.py index a9cdd70de0125..e4205bc50768a 100644 --- a/tests/system/providers/opsgenie/example_opsgenie_notifier.py +++ b/tests/system/providers/opsgenie/example_opsgenie_notifier.py @@ -21,8 +21,8 @@ from datetime import datetime from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.opsgenie.notifications.opsgenie import send_opsgenie_notification +from airflow.providers.standard.core.operators.bash import BashOperator with DAG( "opsgenie_notifier", diff --git a/tests/system/providers/singularity/example_singularity.py b/tests/system/providers/singularity/example_singularity.py index d802fbb31e820..74f4346e493cb 100644 --- a/tests/system/providers/singularity/example_singularity.py +++ b/tests/system/providers/singularity/example_singularity.py @@ -21,8 +21,8 @@ from datetime import datetime, timedelta from airflow import DAG -from airflow.operators.bash import BashOperator from airflow.providers.singularity.operators.singularity import SingularityOperator +from airflow.providers.standard.core.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "singularity_sample" diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index 5cb52696f19ce..eafd23c57128b 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -23,9 +23,9 @@ import pytest from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.dag_dependency import DagDependency from airflow.utils import dot_renderer, timezone from airflow.utils.state import State diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index 084d8c35ac03e..6ba94f85b31aa 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -34,9 +34,9 @@ from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.dag_edges import dag_edges from airflow.utils.task_group import TASKGROUP_ARGS_EXPECTED_TYPES, TaskGroup, task_group_to_dict from tests.models import DEFAULT_DATE diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index f3947b141a347..3eb6dbf4f787b 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -28,8 +28,8 @@ from airflow.models.dag import DAG from airflow.models.renderedtifields import RenderedTaskInstanceFields from airflow.models.variable import Variable -from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone from airflow.utils.session import create_session diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 7b65051724c27..d3ba9d33636fe 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -34,9 +34,9 @@ from airflow.models.taskinstance import TaskInstance from airflow.models.taskreschedule import TaskReschedule from airflow.models.xcom import XCom -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.providers.celery.executors.celery_executor import CeleryExecutor +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.security import permissions from airflow.utils import timezone from airflow.utils.log.logging_mixin import ExternalLoggingMixin From 1480721401b7ada10b9670b5bb518544ca2c46fe Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Wed, 18 Sep 2024 20:07:57 +0100 Subject: [PATCH 16/48] updated airflow version and resolved rebasing failures --- .../celery/executors/celery_executor_utils.py | 2 -- airflow/providers/standard/provider.yaml | 9 +++------ generated/provider_dependencies.json | 12 +----------- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index 359b657c0f075..a8d9037b3a325 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -109,8 +109,6 @@ def on_celery_import_modules(*args, **kwargs): """ import jinja2.ext # noqa: F401 - import airflow.operators.python # noqa: F401 - with contextlib.suppress(ImportError): import numpy # noqa: F401 diff --git a/airflow/providers/standard/provider.yaml b/airflow/providers/standard/provider.yaml index cbc67aeb33c1f..615af7e6ccfb3 100644 --- a/airflow/providers/standard/provider.yaml +++ b/airflow/providers/standard/provider.yaml @@ -28,7 +28,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.10.0 integrations: - integration-name: Standard @@ -37,16 +37,12 @@ integrations: how-to-guide: - /docs/apache-airflow-providers-standard/operators.rst -integrations: - - integration-name: Standard - external-doc-url: https://airflow.apache.org/ - tags: [apache] - operators: - integration-name: Standard python-modules: - airflow.providers.standard.operators.datetime - airflow.providers.standard.operators.weekday + - airflow.providers.standard.core.operators.bash sensors: - integration-name: Standard @@ -55,3 +51,4 @@ sensors: - airflow.providers.standard.sensors.time_delta - airflow.providers.standard.sensors.time - airflow.providers.standard.sensors.weekday + - airflow.providers.standard.core.sensors.bash diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 4158d2d95e622..b1ff0f7dcfab7 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -550,16 +550,6 @@ "excluded-python-versions": [], "state": "ready" }, - "essentials": { - "deps": [ - "apache-airflow>=2.8.0" - ], - "devel-deps": [], - "plugins": [], - "cross-providers-deps": [], - "excluded-python-versions": [], - "state": "ready" - }, "exasol": { "deps": [ "apache-airflow-providers-common-sql>=1.14.1", @@ -1283,7 +1273,7 @@ }, "standard": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.10.0" ], "devel-deps": [], "plugins": [], From 6cad91e91ce1b80b9db227a4258a14a0392526c8 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Wed, 18 Sep 2024 22:57:26 +0100 Subject: [PATCH 17/48] fix selective check tests for bash --- dev/breeze/tests/test_selective_checks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 61b8f61a0609c..b6c553c964902 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -705,7 +705,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("airflow/providers/standard/core/operators/bash.py",), { - "affected-providers-list-as-string": None, + "affected-providers-list-as-string": "standard", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "python-versions": "['3.9']", @@ -717,14 +717,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "false", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always Core Operators Serialization", + "parallel-test-types-list-as-string": "Always Core Providers[standard] Serialization", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-folders": "['providers']", }, - id="Force Core and Serialization tests to run when airflow bash.py changed", + id="Providers tests and Serialization tests to run when airflow bash.py changed", ), pytest.param( ("tests/operators/bash.py",), From 079c4bb3dbaf1d6ca4777e820a0aa3ec87f58cc7 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 01:00:04 +0100 Subject: [PATCH 18/48] Added BashOperator in compat.py to work with old imports --- airflow/example_dags/example_sensors.py | 4 ++-- .../kubernetes/executors/test_kubernetes_executor.py | 2 +- .../providers/cncf/kubernetes/test_template_rendering.py | 2 +- tests/providers/openlineage/extractors/test_bash.py | 2 +- tests/providers/openlineage/extractors/test_python.py | 2 +- tests/providers/openlineage/plugins/test_adapter.py | 2 +- tests/providers/openlineage/plugins/test_utils.py | 3 +-- tests/sensors/test_external_task_sensor.py | 2 +- tests/test_utils/compat.py | 9 ++++++++- 9 files changed, 17 insertions(+), 11 deletions(-) diff --git a/airflow/example_dags/example_sensors.py b/airflow/example_dags/example_sensors.py index 6fb564e63ae43..782408f3928e7 100644 --- a/airflow/example_dags/example_sensors.py +++ b/airflow/example_dags/example_sensors.py @@ -22,11 +22,11 @@ import pendulum from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.standard.sensors.time import TimeSensor, TimeSensorAsync from airflow.providers.standard.sensors.time_delta import TimeDeltaSensor, TimeDeltaSensorAsync from airflow.providers.standard.sensors.weekday import DayOfWeekSensor -from airflow.sensors.bash import BashSensor +from airflow.providers.standard.core.sensors.bash import BashSensor from airflow.sensors.filesystem import FileSensor from airflow.sensors.python import PythonSensor from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py index 46ddd61276fe8..12435426dd899 100644 --- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -52,9 +52,9 @@ get_logs_task_metadata, ) from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State, TaskInstanceState +from tests.test_utils.compat import BashOperator from tests.test_utils.config import conf_vars pytestmark = pytest.mark.skip_if_database_isolation_mode diff --git a/tests/providers/cncf/kubernetes/test_template_rendering.py b/tests/providers/cncf/kubernetes/test_template_rendering.py index 2b4644bf6dc06..4c087d6040e68 100644 --- a/tests/providers/cncf/kubernetes/test_template_rendering.py +++ b/tests/providers/cncf/kubernetes/test_template_rendering.py @@ -25,10 +25,10 @@ from airflow.configuration import TEST_DAGS_FOLDER from airflow.models.renderedtifields import RenderedTaskInstanceFields, RenderedTaskInstanceFields as RTIF from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec, render_k8s_pod_yaml -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.session import create_session from airflow.version import version from tests.models import DEFAULT_DATE +from tests.test_utils.compat import BashOperator pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] diff --git a/tests/providers/openlineage/extractors/test_bash.py b/tests/providers/openlineage/extractors/test_bash.py index e33766e57af24..fc862e5ee30b9 100644 --- a/tests/providers/openlineage/extractors/test_bash.py +++ b/tests/providers/openlineage/extractors/test_bash.py @@ -27,7 +27,7 @@ from airflow import DAG from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.openlineage.extractors.bash import BashExtractor -from airflow.providers.standard.core.operators.bash import BashOperator +from tests.test_utils.compat import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/extractors/test_python.py b/tests/providers/openlineage/extractors/test_python.py index a6e73a9c5997d..44c5503b712d8 100644 --- a/tests/providers/openlineage/extractors/test_python.py +++ b/tests/providers/openlineage/extractors/test_python.py @@ -30,7 +30,7 @@ from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.operators.python import PythonOperator from airflow.providers.openlineage.extractors.python import PythonExtractor -from airflow.providers.standard.core.operators.bash import BashOperator +from tests.test_utils.compat import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/plugins/test_adapter.py b/tests/providers/openlineage/plugins/test_adapter.py index 8c1a758371605..b01fe46fdca13 100644 --- a/tests/providers/openlineage/plugins/test_adapter.py +++ b/tests/providers/openlineage/plugins/test_adapter.py @@ -50,8 +50,8 @@ AirflowStateRunFacet, ) from airflow.providers.openlineage.utils.utils import get_airflow_job_facet -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup +from tests.test_utils.compat import BashOperator from tests.test_utils.config import conf_vars pytestmark = pytest.mark.db_test diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py index 61eb2b1b0721a..00c50d08de506 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/tests/providers/openlineage/plugins/test_utils.py @@ -40,11 +40,10 @@ get_fully_qualified_class_name, is_operator_disabled, ) -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.log.secrets_masker import _secrets_masker from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index 3d6268834dce2..c2e2f1f9c6da0 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -35,10 +35,10 @@ from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel from airflow.models.xcom_arg import XComArg -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator from airflow.providers.standard.sensors.time import TimeSensor +from airflow.providers.standard.core.operators.bash import BashOperator from airflow.sensors.external_task import ( ExternalTaskMarker, ExternalTaskSensor, diff --git a/tests/test_utils/compat.py b/tests/test_utils/compat.py index 09f3653db82d8..3f1d22d0a0615 100644 --- a/tests/test_utils/compat.py +++ b/tests/test_utils/compat.py @@ -38,7 +38,6 @@ except ImportError: from airflow.models.errors import ImportError as ParseImportError # type: ignore[no-redef,attr-defined] - from airflow import __version__ as airflow_version AIRFLOW_VERSION = Version(airflow_version) @@ -53,6 +52,14 @@ # Compatibility for Airflow 2.7.* from airflow.models.baseoperator import BaseOperatorLink +try: + from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.core.sensors.bash import BashSensor +except ImportError: + # Compatibility for Airflow < 2.10.* + from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] + from airflow.sensors.bash import BashSensor # type: ignore[no-redef,attr-defined] + if TYPE_CHECKING: from airflow.models.asset import ( From eeac34792fda652243c160b1616b03cc97572615 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 09:08:56 +0100 Subject: [PATCH 19/48] import bash operator with old module --- airflow/example_dags/example_bash_operator.py | 5 ++++- airflow/providers/edge/example_dags/integration_test.py | 6 +++++- generated/provider_dependencies.json | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py index 2883f6c2fb7f3..886291d615733 100644 --- a/airflow/example_dags/example_bash_operator.py +++ b/airflow/example_dags/example_bash_operator.py @@ -25,8 +25,11 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator +try: + from airflow.providers.standard.core.operators.bash import BashOperator +except ImportError: + from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] with DAG( dag_id="example_bash_operator", schedule="0 0 * * *", diff --git a/airflow/providers/edge/example_dags/integration_test.py b/airflow/providers/edge/example_dags/integration_test.py index d6074abd30ddc..73e752e99e58a 100644 --- a/airflow/providers/edge/example_dags/integration_test.py +++ b/airflow/providers/edge/example_dags/integration_test.py @@ -32,10 +32,14 @@ from airflow.models.dag import DAG from airflow.models.param import Param from airflow.models.variable import Variable -from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +try: + from airflow.providers.standard.core.operators.bash import BashOperator +except ImportError: + from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] + with DAG( dag_id="integration_test", dag_display_name="Integration Test", diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index b1ff0f7dcfab7..9e802dd9b35e4 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -532,7 +532,9 @@ "plugin-class": "airflow.providers.edge.plugins.edge_executor_plugin.EdgeExecutorPlugin" } ], - "cross-providers-deps": [], + "cross-providers-deps": [ + "standard" + ], "excluded-python-versions": [], "state": "not-ready" }, From a5d45512aa1c792185fe92a7d3040bea62e638a0 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 11:32:44 +0100 Subject: [PATCH 20/48] update selective checks and imports in test_utils --- airflow/example_dags/example_bash_operator.py | 5 +---- dev/breeze/tests/test_selective_checks.py | 4 ++-- tests/providers/openlineage/utils/test_utils.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py index 886291d615733..2883f6c2fb7f3 100644 --- a/airflow/example_dags/example_bash_operator.py +++ b/airflow/example_dags/example_bash_operator.py @@ -25,11 +25,8 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator +from airflow.providers.standard.core.operators.bash import BashOperator -try: - from airflow.providers.standard.core.operators.bash import BashOperator -except ImportError: - from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] with DAG( dag_id="example_bash_operator", schedule="0 0 * * *", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index b6c553c964902..f1682f04aece6 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -705,7 +705,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("airflow/providers/standard/core/operators/bash.py",), { - "affected-providers-list-as-string": "standard", + "affected-providers-list-as-string": "edge standard", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "python-versions": "['3.9']", @@ -720,7 +720,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always Core Providers[standard] Serialization", + "parallel-test-types-list-as-string": "Always Core Providers[edge, standard] Serialization", "needs-mypy": "true", "mypy-folders": "['providers']", }, diff --git a/tests/providers/openlineage/utils/test_utils.py b/tests/providers/openlineage/utils/test_utils.py index fdad1f41966e2..2a7b89b14c026 100644 --- a/tests/providers/openlineage/utils/test_utils.py +++ b/tests/providers/openlineage/utils/test_utils.py @@ -40,10 +40,10 @@ get_operator_class, get_user_provided_run_facets, ) -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedBaseOperator from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType +from tests.test_utils.compat import BashOperator from tests.test_utils.mock_operators import MockOperator From e444409ccfb19d2eaa52bc7d75e4bcf7cfd3b8a2 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 13:44:23 +0100 Subject: [PATCH 21/48] fix 2.8.4 compat tests --- dev/breeze/tests/test_selective_checks.py | 4 ++-- tests/dags/test_sensor.py | 1 + .../providers/openlineage/utils/test_utils.py | 18 +++++++++++------- tests/test_utils/compat.py | 2 ++ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index f1682f04aece6..39af018562011 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -720,11 +720,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always Core Providers[edge, standard] Serialization", + "parallel-test-types-list-as-string": "Always Core Providers[edge,standard] Serialization", "needs-mypy": "true", "mypy-folders": "['providers']", }, - id="Providers tests and Serialization tests to run when airflow bash.py changed", + id="Providers standard tests and Serialization tests to run when airflow bash.py changed", ), pytest.param( ("tests/operators/bash.py",), diff --git a/tests/dags/test_sensor.py b/tests/dags/test_sensor.py index d023949e31a98..480d81f30e02c 100644 --- a/tests/dags/test_sensor.py +++ b/tests/dags/test_sensor.py @@ -22,6 +22,7 @@ from airflow.models.dag import DAG from airflow.providers.standard.sensors.date_time import DateTimeSensor from airflow.utils import timezone +from tests.test_utils.compat import DateTimeSensor with DAG( dag_id="test_sensor", start_date=datetime.datetime(2022, 1, 1), catchup=False, schedule="@once" diff --git a/tests/providers/openlineage/utils/test_utils.py b/tests/providers/openlineage/utils/test_utils.py index 2a7b89b14c026..d99374075e621 100644 --- a/tests/providers/openlineage/utils/test_utils.py +++ b/tests/providers/openlineage/utils/test_utils.py @@ -43,9 +43,13 @@ from airflow.serialization.serialized_objects import SerializedBaseOperator from airflow.utils.task_group import TaskGroup from airflow.utils.types import DagRunType -from tests.test_utils.compat import BashOperator +from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator from tests.test_utils.mock_operators import MockOperator +BASH_OPERATOR_PATH = "airflow.providers.standard.core.operators.bash.BashOperator" +if not AIRFLOW_V_2_10_PLUS: + BASH_OPERATOR_PATH = "airflow.operators.bash.BashOperator" + class CustomOperatorForTest(BashOperator): pass @@ -82,7 +86,7 @@ def test_get_airflow_job_facet(): }, tasks={ "task_0": { - "operator": "airflow.providers.standard.core.operators.bash.BashOperator", + "operator": BASH_OPERATOR_PATH, "task_group": None, "emits_ol_events": True, "ui_color": "#f0ede4", @@ -166,7 +170,7 @@ def test_get_airflow_dag_run_facet(): def test_get_fully_qualified_class_name_serialized_operator(): - op_module_path = "airflow.providers.standard.core.operators.bash" + op_module_path = BASH_OPERATOR_PATH op_name = "BashOperator" op = BashOperator(task_id="test", bash_command="echo 1") @@ -191,7 +195,7 @@ def test_get_fully_qualified_class_name_mapped_operator(): def test_get_fully_qualified_class_name_bash_operator(): result = get_fully_qualified_class_name(BashOperator(task_id="test", bash_command="echo 0;")) - expected_result = "airflow.providers.standard.core.operators.bash.BashOperator" + expected_result = BASH_OPERATOR_PATH assert result == expected_result @@ -319,7 +323,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_0": { - "operator": "airflow.providers.standard.core.operators.bash.BashOperator", + "operator": BASH_OPERATOR_PATH, "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -360,7 +364,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_3": { - "operator": "airflow.providers.standard.core.operators.bash.BashOperator", + "operator": BASH_OPERATOR_PATH, "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -388,7 +392,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_5": { - "operator": "airflow.providers.standard.core.operators.bash.BashOperator", + "operator": BASH_OPERATOR_PATH, "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, diff --git a/tests/test_utils/compat.py b/tests/test_utils/compat.py index 3f1d22d0a0615..4c49eaecdb677 100644 --- a/tests/test_utils/compat.py +++ b/tests/test_utils/compat.py @@ -55,10 +55,12 @@ try: from airflow.providers.standard.core.operators.bash import BashOperator from airflow.providers.standard.core.sensors.bash import BashSensor + from airflow.providers.standard.time.sensors.date_time import DateTimeSensor except ImportError: # Compatibility for Airflow < 2.10.* from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] from airflow.sensors.bash import BashSensor # type: ignore[no-redef,attr-defined] + from airflow.sensors.date_time import DateTimeSensor # type: ignore[no-redef,attr-defined] if TYPE_CHECKING: From ef21111d45b11e5a4a830c427fdb9202f5ee0ebf Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 13:50:53 +0100 Subject: [PATCH 22/48] add bashoperator import from compat to miscellaneous_test_dag --- tests/dags/test_miscellaneous.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dags/test_miscellaneous.py b/tests/dags/test_miscellaneous.py index ccdffa0dd2275..4a2c6b56a365d 100644 --- a/tests/dags/test_miscellaneous.py +++ b/tests/dags/test_miscellaneous.py @@ -23,7 +23,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from tests.test_utils.compat import BashOperator args = { "owner": "airflow", From 4fae08c4c4d737ad0bbce392f6d3e9c289287481 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 20:38:37 +0100 Subject: [PATCH 23/48] fix openlineage test utils test --- airflow/example_dags/example_bash_operator.py | 6 +++++- tests/providers/openlineage/utils/test_utils.py | 14 +++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py index 2883f6c2fb7f3..e6041f4d2eabe 100644 --- a/airflow/example_dags/example_bash_operator.py +++ b/airflow/example_dags/example_bash_operator.py @@ -25,7 +25,11 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator + +try: + from airflow.providers.standard.core.operators.bash import BashOperator +except ImportError: + from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] with DAG( dag_id="example_bash_operator", diff --git a/tests/providers/openlineage/utils/test_utils.py b/tests/providers/openlineage/utils/test_utils.py index d99374075e621..4de246e71963a 100644 --- a/tests/providers/openlineage/utils/test_utils.py +++ b/tests/providers/openlineage/utils/test_utils.py @@ -46,9 +46,9 @@ from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator from tests.test_utils.mock_operators import MockOperator -BASH_OPERATOR_PATH = "airflow.providers.standard.core.operators.bash.BashOperator" +BASH_OPERATOR_PATH = "airflow.providers.standard.core.operators.bash" if not AIRFLOW_V_2_10_PLUS: - BASH_OPERATOR_PATH = "airflow.operators.bash.BashOperator" + BASH_OPERATOR_PATH = "airflow.operators.bash" class CustomOperatorForTest(BashOperator): @@ -86,7 +86,7 @@ def test_get_airflow_job_facet(): }, tasks={ "task_0": { - "operator": BASH_OPERATOR_PATH, + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": "#f0ede4", @@ -195,7 +195,7 @@ def test_get_fully_qualified_class_name_mapped_operator(): def test_get_fully_qualified_class_name_bash_operator(): result = get_fully_qualified_class_name(BashOperator(task_id="test", bash_command="echo 0;")) - expected_result = BASH_OPERATOR_PATH + expected_result = f"{BASH_OPERATOR_PATH}.BashOperator" assert result == expected_result @@ -323,7 +323,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_0": { - "operator": BASH_OPERATOR_PATH, + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -364,7 +364,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_3": { - "operator": BASH_OPERATOR_PATH, + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, @@ -392,7 +392,7 @@ def sum_values(values: list[int]) -> int: ], }, "task_5": { - "operator": BASH_OPERATOR_PATH, + "operator": f"{BASH_OPERATOR_PATH}.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": BashOperator.ui_color, From 1aa32b5783956ee40c805f68b805c402511c6a38 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 20:48:33 +0100 Subject: [PATCH 24/48] revert pre-commit removed imports --- .../providers/celery/executors/celery_executor_utils.py | 9 +++++++++ generated/provider_dependencies.json | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index a8d9037b3a325..1c23b5601d4b8 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -109,6 +109,15 @@ def on_celery_import_modules(*args, **kwargs): """ import jinja2.ext # noqa: F401 + import airflow.jobs.local_task_job_runner + import airflow.macros + import airflow.operators.python + + try: + import airflow.providers.standard.core.operators.bash + except ImportError: + import airflow.operators.bash # noqa: F401 + with contextlib.suppress(ImportError): import numpy # noqa: F401 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 9e802dd9b35e4..57bca636ed605 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -344,7 +344,8 @@ "devel-deps": [], "plugins": [], "cross-providers-deps": [ - "cncf.kubernetes" + "cncf.kubernetes", + "standard" ], "excluded-python-versions": [], "state": "ready" From 4439931c06cdc6a5696f26675f08436b99a72df7 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 21:01:13 +0100 Subject: [PATCH 25/48] remove example_branch_datetime_operator from exclusions in pre-commit hook check --- .pre-commit-config.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e4d8886b55326..a163fc0f4cfc4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -965,10 +965,7 @@ repos: entry: ./scripts/ci/pre_commit/check_system_tests.py language: python files: ^tests/system/.*/example_[^/]*\.py$ - exclude: > - (?x) - ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ | - ^tests/system/providers/standard/time/operators/example_branch_datetime_operator.py + exclude: ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$ pass_filenames: true additional_dependencies: ['rich>=12.4.4'] - id: generate-pypi-readme From 1496477aa443541b507afd524e838478c905f0f8 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 19 Sep 2024 21:17:54 +0100 Subject: [PATCH 26/48] import bashoperator using test utils compat in views/utils --- dev/breeze/tests/test_selective_checks.py | 4 ++-- tests/utils/test_dot_renderer.py | 2 +- tests/utils/test_task_group.py | 2 +- tests/www/views/test_views_rendered.py | 3 +-- tests/www/views/test_views_tasks.py | 4 ++-- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 39af018562011..014e6fbb97355 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -705,7 +705,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("airflow/providers/standard/core/operators/bash.py",), { - "affected-providers-list-as-string": "edge standard", + "affected-providers-list-as-string": "celery edge standard", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "python-versions": "['3.9']", @@ -1720,7 +1720,7 @@ def test_upgrade_to_newer_dependencies( ), pytest.param( ("airflow/providers/celery/file.py",), - {"docs-list-as-string": "apache-airflow celery cncf.kubernetes"}, + {"docs-list-as-string": "apache-airflow celery cncf.kubernetes standard"}, id="Celery python files changed", ), pytest.param( diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index eafd23c57128b..0376848fce829 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -25,11 +25,11 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.dag_dependency import DagDependency from airflow.utils import dot_renderer, timezone from airflow.utils.state import State from airflow.utils.task_group import TaskGroup +from tests.test_utils.compat import BashOperator from tests.test_utils.db import clear_db_dags START_DATE = timezone.utcnow() diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index 6ba94f85b31aa..a6008dc58c03f 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -36,10 +36,10 @@ from airflow.models.xcom_arg import XComArg from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.utils.dag_edges import dag_edges from airflow.utils.task_group import TASKGROUP_ARGS_EXPECTED_TYPES, TaskGroup, task_group_to_dict from tests.models import DEFAULT_DATE +from tests.test_utils.compat import BashOperator def make_task(name, type_="classic"): diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 3eb6dbf4f787b..2d1754af29f65 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -29,14 +29,13 @@ from airflow.models.renderedtifields import RenderedTaskInstanceFields from airflow.models.variable import Variable from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import DagRunState, TaskInstanceState from airflow.utils.types import DagRunType from tests.conftest import initial_db_init -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields from tests.test_utils.www import check_content_in_response, check_content_not_in_response diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index d3ba9d33636fe..85ce160aca8f5 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -36,7 +36,6 @@ from airflow.models.xcom import XCom from airflow.operators.empty import EmptyOperator from airflow.providers.celery.executors.celery_executor import CeleryExecutor -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.security import permissions from airflow.utils import timezone from airflow.utils.log.logging_mixin import ExternalLoggingMixin @@ -44,12 +43,13 @@ from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType from airflow.www.views import TaskInstanceModelView, _safe_parse_datetime +from tests.test_utils.api_connexion_utils import create_user, delete_roles, delete_user from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, delete_user, ) -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS +from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator from tests.test_utils.config import conf_vars from tests.test_utils.db import clear_db_runs, clear_db_xcom from tests.test_utils.www import check_content_in_response, check_content_not_in_response, client_with_login From a66b2dd20302ea33ba427285858759bb2c378242 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 20 Sep 2024 11:02:06 +0100 Subject: [PATCH 27/48] fix openlineage tests for compat and selective checks --- airflow/example_dags/example_bash_operator.py | 6 +----- dev/breeze/tests/test_selective_checks.py | 2 +- tests/providers/openlineage/plugins/test_utils.py | 12 +++++++----- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py index e6041f4d2eabe..2883f6c2fb7f3 100644 --- a/airflow/example_dags/example_bash_operator.py +++ b/airflow/example_dags/example_bash_operator.py @@ -25,11 +25,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator - -try: - from airflow.providers.standard.core.operators.bash import BashOperator -except ImportError: - from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] +from airflow.providers.standard.core.operators.bash import BashOperator with DAG( dag_id="example_bash_operator", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 014e6fbb97355..027f97c868e07 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -720,7 +720,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always Core Providers[edge,standard] Serialization", + "parallel-test-types-list-as-string": "Always Core Providers[celery,edge,standard] Serialization", "needs-mypy": "true", "mypy-folders": "['providers']", }, diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py index 00c50d08de506..830c528db804a 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/tests/providers/openlineage/plugins/test_utils.py @@ -43,11 +43,15 @@ from airflow.utils import timezone from airflow.utils.log.secrets_masker import _secrets_masker from airflow.utils.state import State -from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator +from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, BashOperator if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType +BASH_OPERATOR_PATH = "airflow.providers.standard.core.operators.bash" +if not AIRFLOW_V_2_10_PLUS: + BASH_OPERATOR_PATH = "airflow.operators.bash" + class SafeStrDict(dict): def __str__(self): @@ -261,7 +265,7 @@ def test_get_fully_qualified_class_name(): from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter result = get_fully_qualified_class_name(BashOperator(task_id="test", bash_command="exit 0;")) - assert result == "airflow.providers.standard.core.operators.bash.BashOperator" + assert result == f"{BASH_OPERATOR_PATH}.BashOperator" result = get_fully_qualified_class_name(OpenLineageAdapter()) assert result == "airflow.providers.openlineage.plugins.adapter.OpenLineageAdapter" @@ -277,7 +281,7 @@ def test_is_operator_disabled(mock_disabled_operators): assert is_operator_disabled(op) is False mock_disabled_operators.return_value = { - "airflow.providers.standard.core.operators.bash.BashOperator", + f"{BASH_OPERATOR_PATH}.BashOperator", "airflow.operators.python.PythonOperator", } assert is_operator_disabled(op) is True @@ -302,8 +306,6 @@ def test_includes_full_task_info(mock_include_full_task_info): @patch("airflow.providers.openlineage.conf.include_full_task_info") def test_does_not_include_full_task_info(mock_include_full_task_info): - from airflow.providers.standard.core.operators.bash import BashOperator - mock_include_full_task_info.return_value = False # There should be no 'bash_command' in excludes and it's not in includes - so # it's a good choice for checking TaskInfo vs TaskInfoComplete From 5f987f738ee4e925900b2c9fe31156dc2c9baa2d Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 20 Sep 2024 14:10:51 +0100 Subject: [PATCH 28/48] add standard provider package build in basic-tests.yml to test python client --- .github/workflows/basic-tests.yml | 5 +++++ airflow/providers/celery/executors/celery_executor_utils.py | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 2ccb239487679..49d6a7245bc88 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -200,6 +200,11 @@ jobs: breeze release-management prepare-provider-packages fab --package-format wheel --skip-tag-check - name: "Install Airflow with fab for webserver tests" run: pip install . dist/apache_airflow_providers_fab-*.whl + - name: "Prepare Standard provider packages: wheel" + run: > + breeze release-management prepare-provider-packages standard --package-format wheel --skip-tag-check + - name: "Install Airflow with standard provider for webserver tests" + run: pip install . dist/apache_airflow_providers_standard-*.whl - name: "Install Python client" run: pip install ./dist/apache_airflow_client-*.whl - name: "Initialize Airflow DB and start webserver" diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index 1c23b5601d4b8..bfb3863a320b8 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -112,11 +112,7 @@ def on_celery_import_modules(*args, **kwargs): import airflow.jobs.local_task_job_runner import airflow.macros import airflow.operators.python - - try: - import airflow.providers.standard.core.operators.bash - except ImportError: - import airflow.operators.bash # noqa: F401 + import airflow.providers.standard.core.operators.bash # noqa: F401 with contextlib.suppress(ImportError): import numpy # noqa: F401 From d0dec46dac4b9f6b2e658ea723d0b49a80a0a90c Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 20 Sep 2024 21:08:59 +0100 Subject: [PATCH 29/48] copy the standard provider package into the k8s image build to enable it to work with kubernetes tests --- dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index c6cc343ae4aee..b0869d4e6da13 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -606,7 +606,7 @@ def _rebuild_k8s_image( {extra_copy_command} COPY --chown=airflow:0 airflow/example_dags/ /opt/airflow/dags/ - +COPY --chown=airflow:0 airflow/providers/standard/ /opt/airflow/airflow/providers/standard/ COPY --chown=airflow:0 airflow/providers/cncf/kubernetes/kubernetes_executor_templates/ /opt/airflow/pod_templates/ ENV GUNICORN_CMD_ARGS='--preload' AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL=0 From eda4101ea6b60df3dec634da9686bd25f34ce930 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sat, 21 Sep 2024 21:04:27 +0100 Subject: [PATCH 30/48] remove standard provider copy command from kubernetes commands --- dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index b0869d4e6da13..c6cc343ae4aee 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -606,7 +606,7 @@ def _rebuild_k8s_image( {extra_copy_command} COPY --chown=airflow:0 airflow/example_dags/ /opt/airflow/dags/ -COPY --chown=airflow:0 airflow/providers/standard/ /opt/airflow/airflow/providers/standard/ + COPY --chown=airflow:0 airflow/providers/cncf/kubernetes/kubernetes_executor_templates/ /opt/airflow/pod_templates/ ENV GUNICORN_CMD_ARGS='--preload' AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL=0 From 78fb004857c038170538179e83e98f424cf13d3f Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 22 Sep 2024 00:11:00 +0100 Subject: [PATCH 31/48] remove core folder from standard provider --- .pre-commit-config.yaml | 2 +- airflow/decorators/bash.py | 2 +- airflow/example_dags/example_bash_operator.py | 2 +- airflow/example_dags/example_complex.py | 2 +- .../example_dags/example_inlet_event_extra.py | 2 +- .../example_dags/example_outlet_event_extra.py | 2 +- .../example_passing_params_via_test_command.py | 2 +- airflow/example_dags/example_sensors.py | 4 ++-- airflow/example_dags/example_setup_teardown.py | 2 +- airflow/example_dags/example_task_group.py | 2 +- .../example_dags/example_trigger_target_dag.py | 2 +- airflow/example_dags/example_xcom.py | 2 +- airflow/example_dags/example_xcomargs.py | 2 +- airflow/example_dags/tutorial.py | 2 +- .../celery/executors/celery_executor_utils.py | 5 +---- .../edge/example_dags/integration_test.py | 2 +- airflow/providers/openlineage/provider.yaml | 2 +- airflow/providers/standard/core/__init__.py | 16 ---------------- .../standard/core/operators/__init__.py | 16 ---------------- .../providers/standard/core/sensors/__init__.py | 16 ---------------- .../standard/{core => }/operators/bash.py | 0 airflow/providers/standard/provider.yaml | 4 ++-- .../standard/{core => }/sensors/bash.py | 0 dev/breeze/tests/test_selective_checks.py | 2 +- dev/perf/dags/elastic_dag.py | 2 +- dev/perf/dags/perf_dag_2.py | 2 +- .../notifications/chime_notifier_howto_guide.rst | 2 +- .../notifications/sns.rst | 2 +- .../notifications/sqs.rst | 2 +- .../apprise_notifier_howto_guide.rst | 2 +- .../notifications/jira-notifier-howto-guide.rst | 2 +- .../operators/cloud/mlengine.rst | 2 +- .../operators/cloud/pubsub.rst | 2 +- .../guides/developer.rst | 2 +- .../guides/user.rst | 4 ++-- .../pagerduty_notifier_howto_guide.rst | 2 +- .../notifications/slack_notifier_howto_guide.rst | 2 +- .../slackwebhook_notifier_howto_guide.rst | 2 +- .../notifications/smtp_notifier_howto_guide.rst | 2 +- .../administration-and-deployment/lineage.rst | 2 +- docs/apache-airflow/best-practices.rst | 2 +- docs/apache-airflow/core-concepts/dag-run.rst | 4 ++-- docs/apache-airflow/core-concepts/dags.rst | 2 +- docs/apache-airflow/core-concepts/operators.rst | 2 +- docs/apache-airflow/core-concepts/tasks.rst | 2 +- docs/apache-airflow/howto/notifications.rst | 2 +- docs/apache-airflow/howto/operator/bash.rst | 4 ++-- docs/apache-airflow/index.rst | 2 +- docs/apache-airflow/operators-and-hooks-ref.rst | 4 ++-- docs/apache-airflow/tutorial/taskflow.rst | 2 +- docs/exts/templates/openlineage.rst.jinja2 | 2 +- tests/callbacks/test_callback_requests.py | 6 +++--- tests/cli/commands/test_task_command.py | 2 +- tests/core/test_core.py | 2 +- tests/dags/subdir2/test_dont_ignore_this.py | 2 +- tests/dags/test_assets.py | 2 +- .../test_backfill_with_upstream_failed_task.py | 2 +- tests/dags/test_default_impersonation.py | 2 +- tests/dags/test_example_bash_operator.py | 2 +- tests/dags/test_failing.py | 2 +- tests/dags/test_heartbeat_failed_fast.py | 2 +- tests/dags/test_impersonation.py | 2 +- tests/dags/test_multiple_dags.py | 2 +- tests/dags/test_no_impersonation.py | 2 +- tests/dags/test_on_failure_callback.py | 2 +- tests/dags/test_retry_handling_job.py | 2 +- tests/dags/test_sensor.py | 1 - tests/decorators/test_setup_teardown.py | 2 +- .../executors/test_celery_executor.py | 2 +- tests/jobs/test_scheduler_job.py | 2 +- tests/listeners/test_listeners.py | 2 +- tests/models/test_dag.py | 2 +- tests/models/test_dagrun.py | 2 +- tests/models/test_renderedtifields.py | 2 +- tests/models/test_serialized_dag.py | 2 +- tests/models/test_taskinstance.py | 2 +- tests/models/test_xcom_arg.py | 2 +- .../providers/openlineage/plugins/test_facets.py | 2 +- .../providers/openlineage/plugins/test_utils.py | 2 +- tests/providers/openlineage/utils/test_utils.py | 2 +- .../standard/core/operators/test_bash.py | 2 +- .../providers/standard/core/sensors/test_bash.py | 2 +- tests/sensors/test_external_task_sensor.py | 2 +- tests/serialization/test_dag_serialization.py | 8 ++++---- .../example_external_task_child_deferrable.py | 2 +- .../providers/amazon/aws/example_appflow.py | 2 +- .../providers/amazon/aws/example_http_to_s3.py | 2 +- tests/system/providers/amazon/aws/utils/k8s.py | 2 +- .../providers/apache/hive/example_twitter_dag.py | 2 +- .../providers/apache/iceberg/example_iceberg.py | 2 +- .../cncf/kubernetes/example_kubernetes.py | 2 +- .../cncf/kubernetes/example_kubernetes_async.py | 2 +- tests/system/providers/docker/example_docker.py | 2 +- .../providers/docker/example_docker_copy_data.py | 2 +- .../cloud/bigquery/example_bigquery_dataset.py | 2 +- .../cloud/bigquery/example_bigquery_queries.py | 2 +- .../bigquery/example_bigquery_queries_async.py | 2 +- .../cloud/bigquery/example_bigquery_to_mssql.py | 2 +- .../bigquery/example_bigquery_to_postgres.py | 2 +- .../cloud/cloud_build/example_cloud_build.py | 2 +- .../example_cloud_memorystore_memcached.py | 2 +- .../example_cloud_memorystore_redis.py | 2 +- .../google/cloud/gcs/example_gcs_copy_delete.py | 2 +- .../google/cloud/gcs/example_gcs_to_gcs.py | 2 +- .../google/cloud/gcs/example_mysql_to_gcs.py | 2 +- .../google/cloud/gcs/example_sftp_to_gcs.py | 2 +- .../providers/google/cloud/gcs/example_sheets.py | 2 +- .../example_kubernetes_engine.py | 2 +- .../example_kubernetes_engine_async.py | 2 +- .../google/cloud/ml_engine/example_mlengine.py | 2 +- .../natural_language/example_natural_language.py | 2 +- .../google/cloud/pubsub/example_pubsub.py | 2 +- .../cloud/sql_to_sheets/example_sql_to_sheets.py | 2 +- .../google/cloud/tasks/example_queue.py | 2 +- .../cloud/transfers/example_postgres_to_gcs.py | 2 +- .../google/cloud/translate/example_translate.py | 2 +- .../example_video_intelligence.py | 2 +- .../vision/example_vision_annotate_image.py | 2 +- .../datacatalog/example_datacatalog_entries.py | 2 +- .../example_datacatalog_search_catalog.py | 2 +- .../example_datacatalog_tag_templates.py | 2 +- .../datacatalog/example_datacatalog_tags.py | 2 +- .../opsgenie/example_opsgenie_notifier.py | 2 +- .../providers/singularity/example_singularity.py | 2 +- tests/test_utils/compat.py | 10 ++++------ 125 files changed, 133 insertions(+), 187 deletions(-) delete mode 100644 airflow/providers/standard/core/__init__.py delete mode 100644 airflow/providers/standard/core/operators/__init__.py delete mode 100644 airflow/providers/standard/core/sensors/__init__.py rename airflow/providers/standard/{core => }/operators/bash.py (100%) rename airflow/providers/standard/{core => }/sensors/bash.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a163fc0f4cfc4..951d08f8ab095 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -729,7 +729,7 @@ repos: files: > (?x) ^airflow/providers/.*\.py$ - exclude: ^.*/.*_vendor/|airflow/providers/standard/core/operators/bash.py + exclude: ^.*/.*_vendor/|airflow/providers/standard/operators/bash.py - id: check-get-lineage-collector-providers language: python name: Check providers import hook lineage code from compat diff --git a/airflow/decorators/bash.py b/airflow/decorators/bash.py index 5b6249859e948..44738492da098 100644 --- a/airflow/decorators/bash.py +++ b/airflow/decorators/bash.py @@ -21,7 +21,7 @@ from typing import Any, Callable, Collection, Mapping, Sequence from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.context import Context, context_merge from airflow.utils.operator_helpers import determine_kwargs from airflow.utils.types import NOTSET diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py index 2883f6c2fb7f3..27702d4cb5f10 100644 --- a/airflow/example_dags/example_bash_operator.py +++ b/airflow/example_dags/example_bash_operator.py @@ -25,7 +25,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator with DAG( dag_id="example_bash_operator", diff --git a/airflow/example_dags/example_complex.py b/airflow/example_dags/example_complex.py index 049d36c2ef120..6d7d504f13d51 100644 --- a/airflow/example_dags/example_complex.py +++ b/airflow/example_dags/example_complex.py @@ -25,7 +25,7 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator with DAG( dag_id="example_complex", diff --git a/airflow/example_dags/example_inlet_event_extra.py b/airflow/example_dags/example_inlet_event_extra.py index 55fb24f16e190..9773df7a3f913 100644 --- a/airflow/example_dags/example_inlet_event_extra.py +++ b/airflow/example_dags/example_inlet_event_extra.py @@ -28,7 +28,7 @@ from airflow.assets import Asset from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator asset = Asset("s3://output/1.txt") diff --git a/airflow/example_dags/example_outlet_event_extra.py b/airflow/example_dags/example_outlet_event_extra.py index fc16809bfa4b7..0d097eab0ac27 100644 --- a/airflow/example_dags/example_outlet_event_extra.py +++ b/airflow/example_dags/example_outlet_event_extra.py @@ -29,7 +29,7 @@ from airflow.assets.metadata import Metadata from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator ds = Asset("s3://output/1.txt") diff --git a/airflow/example_dags/example_passing_params_via_test_command.py b/airflow/example_dags/example_passing_params_via_test_command.py index 3bdb9a25540a4..7dcd963c09681 100644 --- a/airflow/example_dags/example_passing_params_via_test_command.py +++ b/airflow/example_dags/example_passing_params_via_test_command.py @@ -27,7 +27,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator @task(task_id="run_this") diff --git a/airflow/example_dags/example_sensors.py b/airflow/example_dags/example_sensors.py index 782408f3928e7..f639083858101 100644 --- a/airflow/example_dags/example_sensors.py +++ b/airflow/example_dags/example_sensors.py @@ -22,11 +22,11 @@ import pendulum from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator +from airflow.providers.standard.sensors.bash import BashSensor from airflow.providers.standard.sensors.time import TimeSensor, TimeSensorAsync from airflow.providers.standard.sensors.time_delta import TimeDeltaSensor, TimeDeltaSensorAsync from airflow.providers.standard.sensors.weekday import DayOfWeekSensor -from airflow.providers.standard.core.sensors.bash import BashSensor from airflow.sensors.filesystem import FileSensor from airflow.sensors.python import PythonSensor from airflow.utils.trigger_rule import TriggerRule diff --git a/airflow/example_dags/example_setup_teardown.py b/airflow/example_dags/example_setup_teardown.py index b974931ba1b2f..81994fabc202d 100644 --- a/airflow/example_dags/example_setup_teardown.py +++ b/airflow/example_dags/example_setup_teardown.py @@ -22,7 +22,7 @@ import pendulum from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup with DAG( diff --git a/airflow/example_dags/example_task_group.py b/airflow/example_dags/example_task_group.py index 487807a88f1f9..5129ad3cc61e1 100644 --- a/airflow/example_dags/example_task_group.py +++ b/airflow/example_dags/example_task_group.py @@ -23,7 +23,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_group import TaskGroup # [START howto_task_group] diff --git a/airflow/example_dags/example_trigger_target_dag.py b/airflow/example_dags/example_trigger_target_dag.py index 793af39f12593..3af68a25607a4 100644 --- a/airflow/example_dags/example_trigger_target_dag.py +++ b/airflow/example_dags/example_trigger_target_dag.py @@ -27,7 +27,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator @task(task_id="run_this") diff --git a/airflow/example_dags/example_xcom.py b/airflow/example_dags/example_xcom.py index ed6c2f427881b..2563eda77ee19 100644 --- a/airflow/example_dags/example_xcom.py +++ b/airflow/example_dags/example_xcom.py @@ -24,7 +24,7 @@ from airflow.decorators import task from airflow.models.dag import DAG from airflow.models.xcom_arg import XComArg -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator value_1 = [1, 2, 3] value_2 = {"a": "b"} diff --git a/airflow/example_dags/example_xcomargs.py b/airflow/example_dags/example_xcomargs.py index fd373c79dbd86..a7103dc191135 100644 --- a/airflow/example_dags/example_xcomargs.py +++ b/airflow/example_dags/example_xcomargs.py @@ -25,7 +25,7 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator log = logging.getLogger(__name__) diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py index 707a668472ff8..6e27bbcd2e5fe 100644 --- a/airflow/example_dags/tutorial.py +++ b/airflow/example_dags/tutorial.py @@ -32,7 +32,7 @@ from airflow.models.dag import DAG # Operators; we need this to operate! -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator # [END import_module] diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index bfb3863a320b8..58b11169b9be2 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -109,10 +109,7 @@ def on_celery_import_modules(*args, **kwargs): """ import jinja2.ext # noqa: F401 - import airflow.jobs.local_task_job_runner - import airflow.macros - import airflow.operators.python - import airflow.providers.standard.core.operators.bash # noqa: F401 + import airflow.providers.standard.operators.bash # noqa: F401 with contextlib.suppress(ImportError): import numpy # noqa: F401 diff --git a/airflow/providers/edge/example_dags/integration_test.py b/airflow/providers/edge/example_dags/integration_test.py index 73e752e99e58a..0aad61d354c05 100644 --- a/airflow/providers/edge/example_dags/integration_test.py +++ b/airflow/providers/edge/example_dags/integration_test.py @@ -36,7 +36,7 @@ from airflow.operators.python import PythonOperator try: - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator except ImportError: from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index dc67458b3f474..5b08ac6a77ae7 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -84,7 +84,7 @@ config: Exclude some Operators from emitting OpenLineage events by passing a string of semicolon separated full import paths of Operators to disable. type: string - example: "airflow.providers.standard.core.operators.bash.BashOperator; + example: "airflow.providers.standard.operators.bash.BashOperator; airflow.operators.python.PythonOperator" default: "" version_added: 1.1.0 diff --git a/airflow/providers/standard/core/__init__.py b/airflow/providers/standard/core/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/airflow/providers/standard/core/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/standard/core/operators/__init__.py b/airflow/providers/standard/core/operators/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/airflow/providers/standard/core/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/standard/core/sensors/__init__.py b/airflow/providers/standard/core/sensors/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/airflow/providers/standard/core/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/standard/core/operators/bash.py b/airflow/providers/standard/operators/bash.py similarity index 100% rename from airflow/providers/standard/core/operators/bash.py rename to airflow/providers/standard/operators/bash.py diff --git a/airflow/providers/standard/provider.yaml b/airflow/providers/standard/provider.yaml index 615af7e6ccfb3..2d4c4f29bef5c 100644 --- a/airflow/providers/standard/provider.yaml +++ b/airflow/providers/standard/provider.yaml @@ -42,7 +42,7 @@ operators: python-modules: - airflow.providers.standard.operators.datetime - airflow.providers.standard.operators.weekday - - airflow.providers.standard.core.operators.bash + - airflow.providers.standard.operators.bash sensors: - integration-name: Standard @@ -51,4 +51,4 @@ sensors: - airflow.providers.standard.sensors.time_delta - airflow.providers.standard.sensors.time - airflow.providers.standard.sensors.weekday - - airflow.providers.standard.core.sensors.bash + - airflow.providers.standard.sensors.bash diff --git a/airflow/providers/standard/core/sensors/bash.py b/airflow/providers/standard/sensors/bash.py similarity index 100% rename from airflow/providers/standard/core/sensors/bash.py rename to airflow/providers/standard/sensors/bash.py diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 027f97c868e07..3b58a45ae1b9a 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -703,7 +703,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Only Always and common providers tests should run when only common.io and tests/always changed", ), pytest.param( - ("airflow/providers/standard/core/operators/bash.py",), + ("airflow/providers/standard/operators/bash.py",), { "affected-providers-list-as-string": "celery edge standard", "all-python-versions": "['3.9']", diff --git a/dev/perf/dags/elastic_dag.py b/dev/perf/dags/elastic_dag.py index 9805e71053883..30bfc9acf992b 100644 --- a/dev/perf/dags/elastic_dag.py +++ b/dev/perf/dags/elastic_dag.py @@ -24,7 +24,7 @@ from airflow.models.baseoperator import chain from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator # DAG File used in performance tests. Its shape can be configured by environment variables. RE_TIME_DELTA = re.compile( diff --git a/dev/perf/dags/perf_dag_2.py b/dev/perf/dags/perf_dag_2.py index bc87aa0b433d8..592bbe6087838 100644 --- a/dev/perf/dags/perf_dag_2.py +++ b/dev/perf/dags/perf_dag_2.py @@ -24,7 +24,7 @@ import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator args = { "owner": "airflow", diff --git a/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst b/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst index 6e8c27990766b..e15c3a8c0c8e4 100644 --- a/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.chime import send_chime_notification with DAG( diff --git a/docs/apache-airflow-providers-amazon/notifications/sns.rst b/docs/apache-airflow-providers-amazon/notifications/sns.rst index d2b2b982750d3..262cd966ae418 100644 --- a/docs/apache-airflow-providers-amazon/notifications/sns.rst +++ b/docs/apache-airflow-providers-amazon/notifications/sns.rst @@ -33,7 +33,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.sns import send_sns_notification dag_failure_sns_notification = send_sns_notification( diff --git a/docs/apache-airflow-providers-amazon/notifications/sqs.rst b/docs/apache-airflow-providers-amazon/notifications/sqs.rst index cdbd77856d530..d74a2477d62ca 100644 --- a/docs/apache-airflow-providers-amazon/notifications/sqs.rst +++ b/docs/apache-airflow-providers-amazon/notifications/sqs.rst @@ -33,7 +33,7 @@ Example Code: from datetime import datetime, timezone from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.amazon.aws.notifications.sqs import send_sqs_notification dag_failure_sqs_notification = send_sqs_notification( diff --git a/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst b/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst index 0d9c1769c36cb..2a0aeaaa10764 100644 --- a/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst @@ -30,7 +30,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.apprise.notifications.apprise import send_apprise_notification from apprise import NotifyType diff --git a/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst b/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst index e406a192a937d..a5617b9035de0 100644 --- a/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst +++ b/docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst @@ -31,7 +31,7 @@ Example Code from datetime import datetime from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.atlassian.jira.notifications.jira import send_jira_notification with DAG( diff --git a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst index 91238fc61afe6..f64705e1c267c 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst @@ -96,7 +96,7 @@ instead. You can use :ref:`Jinja templating ` with the ``project_id`` and ``model`` fields to dynamically determine their values. The result are saved to :ref:`XCom `, allowing them to be used by other operators. In this case, the -:class:`~airflow.providers.standard.core.operators.bash.BashOperator` is used to print the model information. +:class:`~airflow.providers.standard.operators.bash.BashOperator` is used to print the model information. .. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py :language: python diff --git a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst index a5245ae563af3..8fb497a14f01e 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/pubsub.rst @@ -101,7 +101,7 @@ Also for this action you can use sensor in the deferrable mode: :start-after: [START howto_operator_gcp_pubsub_pull_message_with_operator] :end-before: [END howto_operator_gcp_pubsub_pull_message_with_operator] -To pull messages from XCom use the :class:`~airflow.providers.standard.core.operators.bash.BashOperator`. +To pull messages from XCom use the :class:`~airflow.providers.standard.operators.bash.BashOperator`. .. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py :language: python diff --git a/docs/apache-airflow-providers-openlineage/guides/developer.rst b/docs/apache-airflow-providers-openlineage/guides/developer.rst index ea179ddf507af..ccab215fc1846 100644 --- a/docs/apache-airflow-providers-openlineage/guides/developer.rst +++ b/docs/apache-airflow-providers-openlineage/guides/developer.rst @@ -390,7 +390,7 @@ An Operator inside the Airflow DAG can be annotated with inlets and outlets like import pendulum from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.lineage.entities import Table, File, Column, User diff --git a/docs/apache-airflow-providers-openlineage/guides/user.rst b/docs/apache-airflow-providers-openlineage/guides/user.rst index 9eb6ec69bae9a..4f95253e1f242 100644 --- a/docs/apache-airflow-providers-openlineage/guides/user.rst +++ b/docs/apache-airflow-providers-openlineage/guides/user.rst @@ -257,13 +257,13 @@ full import paths of Airflow Operators to disable as ``disabled_for_operators`` [openlineage] transport = {"type": "http", "url": "http://example.com:5000", "endpoint": "api/v1/lineage"} - disabled_for_operators = 'airflow.providers.standard.core.operators.bash.BashOperator;airflow.operators.python.PythonOperator' + disabled_for_operators = 'airflow.providers.standard.operators.bash.BashOperator;airflow.operators.python.PythonOperator' ``AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS`` environment variable is an equivalent. .. code-block:: ini - AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.providers.standard.core.operators.bash.BashOperator;airflow.operators.python.PythonOperator' + AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.providers.standard.operators.bash.BashOperator;airflow.operators.python.PythonOperator' Full Task Info ^^^^^^^^^^^^^^ diff --git a/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst b/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst index 88db5ef6732b1..658054bd0a5ec 100644 --- a/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.pagerduty.notifications.pagerduty import send_pagerduty_notification with DAG( diff --git a/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst b/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst index 380e916c75f7d..3b6a1e7879924 100644 --- a/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.slack.notifications.slack import send_slack_notification with DAG( diff --git a/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst b/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst index bf404f77f2d33..e6ef3ab41409c 100644 --- a/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-slack/notifications/slackwebhook_notifier_howto_guide.rst @@ -32,7 +32,7 @@ Example Code: from datetime import datetime, timezone from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.slack.notifications.slack_webhook import send_slack_webhook_notification dag_failure_slack_webhook_notification = send_slack_webhook_notification( diff --git a/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst b/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst index 68a41c1294df8..e47f9e340c93b 100644 --- a/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst +++ b/docs/apache-airflow-providers-smtp/notifications/smtp_notifier_howto_guide.rst @@ -31,7 +31,7 @@ Example Code: from datetime import datetime from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.smtp.notifications.smtp import send_smtp_notification with DAG( diff --git a/docs/apache-airflow/administration-and-deployment/lineage.rst b/docs/apache-airflow/administration-and-deployment/lineage.rst index b2c1664a3a54c..3740e8b56f937 100644 --- a/docs/apache-airflow/administration-and-deployment/lineage.rst +++ b/docs/apache-airflow/administration-and-deployment/lineage.rst @@ -36,7 +36,7 @@ works. from airflow.lineage import AUTO from airflow.lineage.entities import File from airflow.models import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator FILE_CATEGORIES = ["CAT1", "CAT2", "CAT3"] diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst index b11ba19aaced0..466f546ff7147 100644 --- a/docs/apache-airflow/best-practices.rst +++ b/docs/apache-airflow/best-practices.rst @@ -480,7 +480,7 @@ It's easier to grab the concept with an example. Let's say that we have the foll from airflow import DAG from airflow.decorators import task from airflow.exceptions import AirflowException - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule diff --git a/docs/apache-airflow/core-concepts/dag-run.rst b/docs/apache-airflow/core-concepts/dag-run.rst index 7f7d8bec0ed3a..97fd4e28a7b58 100644 --- a/docs/apache-airflow/core-concepts/dag-run.rst +++ b/docs/apache-airflow/core-concepts/dag-run.rst @@ -101,7 +101,7 @@ in the configuration file. When turned off, the scheduler creates a DAG run only https://github.com/apache/airflow/blob/main/airflow/example_dags/tutorial.py """ from airflow.models.dag import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator import datetime import pendulum @@ -241,7 +241,7 @@ Example of a parameterized DAG: import pendulum from airflow import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator dag = DAG( "example_parameterized_dag", diff --git a/docs/apache-airflow/core-concepts/dags.rst b/docs/apache-airflow/core-concepts/dags.rst index b0bf1181d276d..64726e08010e1 100644 --- a/docs/apache-airflow/core-concepts/dags.rst +++ b/docs/apache-airflow/core-concepts/dags.rst @@ -574,7 +574,7 @@ TaskGroup also supports ``default_args`` like DAG, it will overwrite the ``defau from airflow import DAG from airflow.decorators import task_group - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator with DAG( diff --git a/docs/apache-airflow/core-concepts/operators.rst b/docs/apache-airflow/core-concepts/operators.rst index df746ba3b99a6..6a0825df28773 100644 --- a/docs/apache-airflow/core-concepts/operators.rst +++ b/docs/apache-airflow/core-concepts/operators.rst @@ -28,7 +28,7 @@ An Operator is conceptually a template for a predefined :doc:`Task `, tha Airflow has a very extensive set of operators available, with some built-in to the core or pre-installed providers. Some popular operators from core include: -- :class:`~airflow.providers.standard.core.operators.bash.BashOperator` - executes a bash command +- :class:`~airflow.providers.standard.operators.bash.BashOperator` - executes a bash command - :class:`~airflow.operators.python.PythonOperator` - calls an arbitrary Python function - :class:`~airflow.operators.email.EmailOperator` - sends an email - Use the ``@task`` decorator to execute an arbitrary Python function. It doesn't support rendering jinja templates passed as arguments. diff --git a/docs/apache-airflow/core-concepts/tasks.rst b/docs/apache-airflow/core-concepts/tasks.rst index 92094e3bc4a85..5adfe8be46024 100644 --- a/docs/apache-airflow/core-concepts/tasks.rst +++ b/docs/apache-airflow/core-concepts/tasks.rst @@ -236,7 +236,7 @@ If you'd like to reproduce zombie tasks for development/testing processes, follo .. code-block:: python from airflow.decorators import dag - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from datetime import datetime diff --git a/docs/apache-airflow/howto/notifications.rst b/docs/apache-airflow/howto/notifications.rst index 2a8f547c486af..993a36b389423 100644 --- a/docs/apache-airflow/howto/notifications.rst +++ b/docs/apache-airflow/howto/notifications.rst @@ -59,7 +59,7 @@ Here's an example of using the above notifier: from datetime import datetime from airflow.models.dag import DAG - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator from myprovider.notifier import MyNotifier diff --git a/docs/apache-airflow/howto/operator/bash.rst b/docs/apache-airflow/howto/operator/bash.rst index 21ca26d53b0d3..e4af9bcad6b83 100644 --- a/docs/apache-airflow/howto/operator/bash.rst +++ b/docs/apache-airflow/howto/operator/bash.rst @@ -22,7 +22,7 @@ BashOperator ============ -Use the :class:`~airflow.providers.standard.core.operators.bash.BashOperator` to execute +Use the :class:`~airflow.providers.standard.operators.bash.BashOperator` to execute commands in a `Bash `__ shell. The Bash command or script to execute is determined by: @@ -390,7 +390,7 @@ There are numerous possibilities with this type of pre-execution enrichment. BashSensor ========== -Use the :class:`~airflow.providers.standard.core.sensors.bash.BashSensor` to use arbitrary command for sensing. The command +Use the :class:`~airflow.providers.standard.sensors.bash.BashSensor` to use arbitrary command for sensing. The command should return 0 when it succeeds, any other value otherwise. .. exampleinclude:: /../../airflow/example_dags/example_sensors.py diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst index e4b1dd6ddc05c..38d62ecd04a72 100644 --- a/docs/apache-airflow/index.rst +++ b/docs/apache-airflow/index.rst @@ -41,7 +41,7 @@ Take a look at the following snippet of code: from airflow import DAG from airflow.decorators import task - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator # A DAG represents a workflow, a collection of tasks with DAG(dag_id="demo", start_date=datetime(2022, 1, 1), schedule="0 0 * * *") as dag: diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst index fe12ac2f5d12a..c82a4f3a66d73 100644 --- a/docs/apache-airflow/operators-and-hooks-ref.rst +++ b/docs/apache-airflow/operators-and-hooks-ref.rst @@ -50,7 +50,7 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Operators - Guides - * - :mod:`airflow.providers.standard.core.operators.bash` + * - :mod:`airflow.providers.standard.operators.bash` - :doc:`How to use ` * - :mod:`airflow.operators.branch` @@ -82,7 +82,7 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Sensors - Guides - * - :mod:`airflow.providers.standard.core.sensors.bash` + * - :mod:`airflow.providers.standard.sensors.bash` - :ref:`How to use ` * - :mod:`airflow.sensors.external_task` diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index a8402a26ea0af..aac04f9b53454 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -437,7 +437,7 @@ the parameter value is used. Adding dependencies between decorated and traditional tasks ----------------------------------------------------------- The above tutorial shows how to create dependencies between TaskFlow functions. However, dependencies can also -be set between traditional tasks (such as :class:`~airflow.providers.standard.core.operators.bash.BashOperator` +be set between traditional tasks (such as :class:`~airflow.providers.standard.operators.bash.BashOperator` or :class:`~airflow.sensors.filesystem.FileSensor`) and TaskFlow functions. Building this dependency is shown in the code below: diff --git a/docs/exts/templates/openlineage.rst.jinja2 b/docs/exts/templates/openlineage.rst.jinja2 index 1fc106bebbad8..dfac543cbe7f1 100644 --- a/docs/exts/templates/openlineage.rst.jinja2 +++ b/docs/exts/templates/openlineage.rst.jinja2 @@ -22,7 +22,7 @@ At the moment, two core operators supports OpenLineage. These operators function capable of running any code, which might limit the extent of lineage extraction. - :class:`~airflow.operators.python.PythonOperator` (via :class:`airflow.providers.openlineage.extractors.python.PythonExtractor`) -- :class:`~airflow.providers.standard.core.operators.bash.BashOperator` (via :class:`airflow.providers.openlineage.extractors.bash.BashExtractor`) +- :class:`~airflow.providers.standard.operators.bash.BashOperator` (via :class:`airflow.providers.openlineage.extractors.bash.BashExtractor`) :class:`~airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator` diff --git a/tests/callbacks/test_callback_requests.py b/tests/callbacks/test_callback_requests.py index f995764a9e6f6..7bbe41387750c 100644 --- a/tests/callbacks/test_callback_requests.py +++ b/tests/callbacks/test_callback_requests.py @@ -27,7 +27,7 @@ ) from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType @@ -100,7 +100,7 @@ def test_simple_ti_roundtrip_exec_config_pod(self): from airflow.callbacks.callback_requests import TaskCallbackRequest from airflow.models import TaskInstance from airflow.models.taskinstance import SimpleTaskInstance - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator test_pod = k8s.V1Pod(metadata=k8s.V1ObjectMeta(name="hello", namespace="ns")) op = BashOperator(task_id="hi", executor_config={"pod_override": test_pod}, bash_command="hi") @@ -115,7 +115,7 @@ def test_simple_ti_roundtrip_dates(self, dag_maker): from airflow.callbacks.callback_requests import TaskCallbackRequest from airflow.models import TaskInstance from airflow.models.taskinstance import SimpleTaskInstance - from airflow.providers.standard.core.operators.bash import BashOperator + from airflow.providers.standard.operators.bash import BashOperator with dag_maker(schedule=timedelta(weeks=1), serialized=True): op = BashOperator(task_id="hi", bash_command="hi") diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index ef0b470551585..3397005b80eb0 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -46,7 +46,7 @@ from airflow.models import DagBag, DagRun, Pool, TaskInstance from airflow.models.serialized_dag import SerializedDagModel from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.session import create_session from airflow.utils.state import State, TaskInstanceState diff --git a/tests/core/test_core.py b/tests/core/test_core.py index eb7702370af5a..d44235f955243 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -29,7 +29,7 @@ from airflow.models.baseoperator import BaseOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_task_fail diff --git a/tests/dags/subdir2/test_dont_ignore_this.py b/tests/dags/subdir2/test_dont_ignore_this.py index cc25f47587764..07f04293d7d54 100644 --- a/tests/dags/subdir2/test_dont_ignore_this.py +++ b/tests/dags/subdir2/test_dont_ignore_this.py @@ -20,7 +20,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2019, 12, 1) diff --git a/tests/dags/test_assets.py b/tests/dags/test_assets.py index 7d82c27bb42a0..014ae6fd0ca9f 100644 --- a/tests/dags/test_assets.py +++ b/tests/dags/test_assets.py @@ -23,7 +23,7 @@ from airflow.exceptions import AirflowFailException, AirflowSkipException from airflow.models.dag import DAG from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator skip_task_dag_dataset = Asset("s3://dag_with_skip_task/output_1.txt", extra={"hi": "bye"}) fail_task_dag_dataset = Asset("s3://dag_with_fail_task/output_1.txt", extra={"hi": "bye"}) diff --git a/tests/dags/test_backfill_with_upstream_failed_task.py b/tests/dags/test_backfill_with_upstream_failed_task.py index 90c1ec8ee93bf..865b0da4ff426 100644 --- a/tests/dags/test_backfill_with_upstream_failed_task.py +++ b/tests/dags/test_backfill_with_upstream_failed_task.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator dag = DAG( dag_id="test_backfill_with_upstream_failed_task", diff --git a/tests/dags/test_default_impersonation.py b/tests/dags/test_default_impersonation.py index dc3f45c16e078..4bee30457b7bd 100644 --- a/tests/dags/test_default_impersonation.py +++ b/tests/dags/test_default_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_example_bash_operator.py b/tests/dags/test_example_bash_operator.py index 55739af8ca3bb..52126f0e10206 100644 --- a/tests/dags/test_example_bash_operator.py +++ b/tests/dags/test_example_bash_operator.py @@ -21,7 +21,7 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator dag = DAG( dag_id="test_example_bash_operator", diff --git a/tests/dags/test_failing.py b/tests/dags/test_failing.py index 58c681dbdbb56..646665a8802ef 100644 --- a/tests/dags/test_failing.py +++ b/tests/dags/test_failing.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator dag = DAG( dag_id="test_failing_bash_operator", diff --git a/tests/dags/test_heartbeat_failed_fast.py b/tests/dags/test_heartbeat_failed_fast.py index 631e37bcf86ef..890756ef2017d 100644 --- a/tests/dags/test_heartbeat_failed_fast.py +++ b/tests/dags/test_heartbeat_failed_fast.py @@ -20,7 +20,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_impersonation.py b/tests/dags/test_impersonation.py index d27d826f88dfc..6c2ca2d810026 100644 --- a/tests/dags/test_impersonation.py +++ b/tests/dags/test_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_multiple_dags.py b/tests/dags/test_multiple_dags.py index d489bc8468553..27f159bfb1272 100644 --- a/tests/dags/test_multiple_dags.py +++ b/tests/dags/test_multiple_dags.py @@ -20,7 +20,7 @@ import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator args = {"owner": "airflow", "retries": 3, "start_date": datetime.datetime(2022, 1, 1)} diff --git a/tests/dags/test_no_impersonation.py b/tests/dags/test_no_impersonation.py index 8d320866a882c..22b47fcc878c8 100644 --- a/tests/dags/test_no_impersonation.py +++ b/tests/dags/test_no_impersonation.py @@ -21,7 +21,7 @@ from datetime import datetime from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_on_failure_callback.py b/tests/dags/test_on_failure_callback.py index 0d130e60a06d7..f6765a3698097 100644 --- a/tests/dags/test_on_failure_callback.py +++ b/tests/dags/test_on_failure_callback.py @@ -22,7 +22,7 @@ from airflow.exceptions import AirflowFailException from airflow.models.dag import DAG from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DEFAULT_DATE = datetime(2016, 1, 1) diff --git a/tests/dags/test_retry_handling_job.py b/tests/dags/test_retry_handling_job.py index e17a084a28503..ede9c4c6ace50 100644 --- a/tests/dags/test_retry_handling_job.py +++ b/tests/dags/test_retry_handling_job.py @@ -20,7 +20,7 @@ from datetime import datetime, timedelta from airflow.models.dag import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator default_args = { "owner": "airflow", diff --git a/tests/dags/test_sensor.py b/tests/dags/test_sensor.py index 480d81f30e02c..07c9cc7efdffb 100644 --- a/tests/dags/test_sensor.py +++ b/tests/dags/test_sensor.py @@ -20,7 +20,6 @@ from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.standard.sensors.date_time import DateTimeSensor from airflow.utils import timezone from tests.test_utils.compat import DateTimeSensor diff --git a/tests/decorators/test_setup_teardown.py b/tests/decorators/test_setup_teardown.py index 13aaed23dc6a1..1f2a3dcdbc630 100644 --- a/tests/decorators/test_setup_teardown.py +++ b/tests/decorators/test_setup_teardown.py @@ -22,7 +22,7 @@ from airflow.decorators import setup, task, task_group, teardown from airflow.decorators.setup_teardown import context_wrapper from airflow.exceptions import AirflowException -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator pytestmark = pytest.mark.db_test diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 8df14eb318eb4..4ec1cc458c3a5 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -43,7 +43,7 @@ from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance from airflow.models.taskinstancekey import TaskInstanceKey -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.state import State, TaskInstanceState from tests.test_utils import db diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 7c5b106bae685..d26369f5d728e 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -61,7 +61,7 @@ from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance, TaskInstanceKey from airflow.operators.empty import EmptyOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone from airflow.utils.file import list_py_file_paths diff --git a/tests/listeners/test_listeners.py b/tests/listeners/test_listeners.py index 9edafd0992ee3..29ec25a9a8d2a 100644 --- a/tests/listeners/test_listeners.py +++ b/tests/listeners/test_listeners.py @@ -25,7 +25,7 @@ from airflow.exceptions import AirflowException from airflow.jobs.job import Job, run_job from airflow.listeners.listener import get_listener_manager -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.session import provide_session from airflow.utils.state import DagRunState, TaskInstanceState diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 4d84198596d06..c79ca24e03a25 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -76,7 +76,7 @@ from airflow.models.taskinstance import TaskInstance as TI from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.security import permissions from airflow.templates import NativeEnvironment, SandboxedEnvironment from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index 9bf50b58412fe..9184f561b3df9 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -38,7 +38,7 @@ from airflow.models.taskreschedule import TaskReschedule from airflow.operators.empty import EmptyOperator from airflow.operators.python import ShortCircuitOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.stats import Stats from airflow.triggers.base import StartTriggerArgs diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index 390c686942e70..1de83954ee4bd 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -31,7 +31,7 @@ from airflow.decorators import task as task_decorator from airflow.models import Variable from airflow.models.renderedtifields import RenderedTaskInstanceFields as RTIF -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.task_instance_session import set_current_task_instance_session from airflow.utils.timezone import datetime from tests.test_utils.asserts import assert_queries_count diff --git a/tests/models/test_serialized_dag.py b/tests/models/test_serialized_dag.py index 7a64d4e4cc1a7..93845e95832cf 100644 --- a/tests/models/test_serialized_dag.py +++ b/tests/models/test_serialized_dag.py @@ -31,7 +31,7 @@ from airflow.models.dagbag import DagBag from airflow.models.dagcode import DagCode from airflow.models.serialized_dag import SerializedDagModel as SDM -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 261c4279c0d3c..c09d3575d1eca 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -78,7 +78,7 @@ from airflow.notifications.basenotifier import BaseNotifier from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.sensors.base import BaseSensorOperator from airflow.sensors.python import PythonSensor from airflow.serialization.serialized_objects import SerializedBaseOperator, SerializedDAG diff --git a/tests/models/test_xcom_arg.py b/tests/models/test_xcom_arg.py index 3e0062e9d24a9..fcc2e546009c9 100644 --- a/tests/models/test_xcom_arg.py +++ b/tests/models/test_xcom_arg.py @@ -20,7 +20,7 @@ from airflow.models.xcom_arg import XComArg from airflow.operators.python import PythonOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.types import NOTSET from tests.test_utils.config import conf_vars from tests.test_utils.db import clear_db_dags, clear_db_runs diff --git a/tests/providers/openlineage/plugins/test_facets.py b/tests/providers/openlineage/plugins/test_facets.py index 5a33b27ced243..d46cadc9d69c6 100644 --- a/tests/providers/openlineage/plugins/test_facets.py +++ b/tests/providers/openlineage/plugins/test_facets.py @@ -81,7 +81,7 @@ def test_airflow_dag_run_facet(): }, tasks={ "task_0": { - "operator": "airflow.providers.standard.core.operators.bash.BashOperator", + "operator": "airflow.providers.standard.operators.bash.BashOperator", "task_group": None, "emits_ol_events": True, "ui_color": "#f0ede4", diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py index 830c528db804a..65874a5ecebf3 100644 --- a/tests/providers/openlineage/plugins/test_utils.py +++ b/tests/providers/openlineage/plugins/test_utils.py @@ -48,7 +48,7 @@ if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType -BASH_OPERATOR_PATH = "airflow.providers.standard.core.operators.bash" +BASH_OPERATOR_PATH = "airflow.providers.standard.operators.bash" if not AIRFLOW_V_2_10_PLUS: BASH_OPERATOR_PATH = "airflow.operators.bash" diff --git a/tests/providers/openlineage/utils/test_utils.py b/tests/providers/openlineage/utils/test_utils.py index 4de246e71963a..20eba76adeb18 100644 --- a/tests/providers/openlineage/utils/test_utils.py +++ b/tests/providers/openlineage/utils/test_utils.py @@ -46,7 +46,7 @@ from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS, BashOperator from tests.test_utils.mock_operators import MockOperator -BASH_OPERATOR_PATH = "airflow.providers.standard.core.operators.bash" +BASH_OPERATOR_PATH = "airflow.providers.standard.operators.bash" if not AIRFLOW_V_2_10_PLUS: BASH_OPERATOR_PATH = "airflow.operators.bash" diff --git a/tests/providers/standard/core/operators/test_bash.py b/tests/providers/standard/core/operators/test_bash.py index 6b579c044daf4..2c29a0b96dc93 100644 --- a/tests/providers/standard/core/operators/test_bash.py +++ b/tests/providers/standard/core/operators/test_bash.py @@ -28,7 +28,7 @@ import pytest from airflow.exceptions import AirflowException, AirflowSkipException, AirflowTaskTimeout -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils import timezone from airflow.utils.state import State from airflow.utils.types import DagRunType diff --git a/tests/providers/standard/core/sensors/test_bash.py b/tests/providers/standard/core/sensors/test_bash.py index 212066498df1c..d51db033be308 100644 --- a/tests/providers/standard/core/sensors/test_bash.py +++ b/tests/providers/standard/core/sensors/test_bash.py @@ -23,7 +23,7 @@ from airflow.exceptions import AirflowFailException, AirflowSensorTimeout from airflow.models.dag import DAG -from airflow.providers.standard.core.sensors.bash import BashSensor +from airflow.providers.standard.sensors.bash import BashSensor class TestBashSensor: diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index c2e2f1f9c6da0..9947a197a0335 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -37,8 +37,8 @@ from airflow.models.xcom_arg import XComArg from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.standard.sensors.time import TimeSensor -from airflow.providers.standard.core.operators.bash import BashOperator from airflow.sensors.external_task import ( ExternalTaskMarker, ExternalTaskSensor, diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 4d5497bd194a4..f0f517042314a 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -62,8 +62,8 @@ from airflow.models.xcom import XCom from airflow.operators.empty import EmptyOperator from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator -from airflow.providers.standard.core.operators.bash import BashOperator -from airflow.providers.standard.core.sensors.bash import BashSensor +from airflow.providers.standard.operators.bash import BashOperator +from airflow.providers.standard.sensors.bash import BashSensor from airflow.security import permissions from airflow.serialization.enums import Encoding from airflow.serialization.json_schema import load_dag_schema_dict @@ -154,7 +154,7 @@ "template_fields_renderers": {"bash_command": "bash", "env": "json"}, "bash_command": "echo {{ task.task_id }}", "_task_type": "BashOperator", - "_task_module": "airflow.providers.standard.core.operators.bash", + "_task_module": "airflow.providers.standard.operators.bash", "pool": "default_pool", "is_setup": False, "is_teardown": False, @@ -2284,7 +2284,7 @@ def test_operator_expand_serde(): "_is_empty": False, "_is_mapped": True, "_needs_expansion": True, - "_task_module": "airflow.providers.standard.core.operators.bash", + "_task_module": "airflow.providers.standard.operators.bash", "_task_type": "BashOperator", "start_trigger_args": None, "start_from_trigger": False, diff --git a/tests/system/core/example_external_task_child_deferrable.py b/tests/system/core/example_external_task_child_deferrable.py index b6feac48d14d2..781ad4ea5ef1f 100644 --- a/tests/system/core/example_external_task_child_deferrable.py +++ b/tests/system/core/example_external_task_child_deferrable.py @@ -19,7 +19,7 @@ from datetime import datetime from airflow import DAG -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator with DAG( dag_id="child_dag", diff --git a/tests/system/providers/amazon/aws/example_appflow.py b/tests/system/providers/amazon/aws/example_appflow.py index 60c18f36be5ea..5ba38533b0211 100644 --- a/tests/system/providers/amazon/aws/example_appflow.py +++ b/tests/system/providers/amazon/aws/example_appflow.py @@ -27,7 +27,7 @@ AppflowRunDailyOperator, AppflowRunFullOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder sys_test_context_task = SystemTestContextBuilder().build() diff --git a/tests/system/providers/amazon/aws/example_http_to_s3.py b/tests/system/providers/amazon/aws/example_http_to_s3.py index 0e6ad0daee0b0..d6424f9802155 100644 --- a/tests/system/providers/amazon/aws/example_http_to_s3.py +++ b/tests/system/providers/amazon/aws/example_http_to_s3.py @@ -25,7 +25,7 @@ from airflow.models.dag import DAG from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator, S3DeleteBucketOperator from airflow.providers.amazon.aws.transfers.http_to_s3 import HttpToS3Operator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder diff --git a/tests/system/providers/amazon/aws/utils/k8s.py b/tests/system/providers/amazon/aws/utils/k8s.py index a27e5eeb27275..a882d9e42842e 100644 --- a/tests/system/providers/amazon/aws/utils/k8s.py +++ b/tests/system/providers/amazon/aws/utils/k8s.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator if TYPE_CHECKING: from airflow.models.operator import Operator diff --git a/tests/system/providers/apache/hive/example_twitter_dag.py b/tests/system/providers/apache/hive/example_twitter_dag.py index fdcd6973402ef..4ceb119ba551c 100644 --- a/tests/system/providers/apache/hive/example_twitter_dag.py +++ b/tests/system/providers/apache/hive/example_twitter_dag.py @@ -27,7 +27,7 @@ from airflow import DAG from airflow.decorators import task from airflow.providers.apache.hive.operators.hive import HiveOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator # -------------------------------------------------------------------------------- # Caveat: This Dag will not run because of missing scripts. diff --git a/tests/system/providers/apache/iceberg/example_iceberg.py b/tests/system/providers/apache/iceberg/example_iceberg.py index 4f144a6344bb8..41e751624b5c0 100644 --- a/tests/system/providers/apache/iceberg/example_iceberg.py +++ b/tests/system/providers/apache/iceberg/example_iceberg.py @@ -20,7 +20,7 @@ from airflow import DAG from airflow.providers.apache.iceberg.hooks.iceberg import IcebergHook -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator bash_command = f""" echo "Our token: {IcebergHook().get_token_macro()}" diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes.py b/tests/system/providers/cncf/kubernetes/example_kubernetes.py index 4220687448633..3756d0c4e21d8 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes.py @@ -29,7 +29,7 @@ from airflow import DAG from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py index 9356fdb9ba269..cb3d25a33fcbc 100644 --- a/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py +++ b/tests/system/providers/cncf/kubernetes/example_kubernetes_async.py @@ -29,7 +29,7 @@ from airflow import DAG from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/tests/system/providers/docker/example_docker.py b/tests/system/providers/docker/example_docker.py index 009b137553edd..18f7d2f0ea0c6 100644 --- a/tests/system/providers/docker/example_docker.py +++ b/tests/system/providers/docker/example_docker.py @@ -22,7 +22,7 @@ from airflow import models from airflow.providers.docker.operators.docker import DockerOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "docker_test" diff --git a/tests/system/providers/docker/example_docker_copy_data.py b/tests/system/providers/docker/example_docker_copy_data.py index 0086bdf59c65d..4e4e8466e501f 100644 --- a/tests/system/providers/docker/example_docker_copy_data.py +++ b/tests/system/providers/docker/example_docker_copy_data.py @@ -34,7 +34,7 @@ from airflow import models from airflow.operators.python import ShortCircuitOperator from airflow.providers.docker.operators.docker import DockerOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "docker_sample_copy_data" diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py index 246e673146e7b..004f996975be1 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_dataset.py @@ -31,7 +31,7 @@ BigQueryGetDatasetOperator, BigQueryUpdateDatasetOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py index 436db58b5bf43..ab7a4b3757b9b 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py @@ -37,7 +37,7 @@ BigQueryTableCheckOperator, BigQueryValueCheckOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py index 410f4debcb3bc..a007e1cd639c0 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries_async.py @@ -35,7 +35,7 @@ BigQueryIntervalCheckOperator, BigQueryValueCheckOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py index bb4b039ffae4c..e9b3269ecfb6c 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_mssql.py @@ -50,7 +50,7 @@ ComputeEngineInsertInstanceOperator, ) from airflow.providers.ssh.operators.ssh import SSHOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py index 541f988d3370b..4a3b0386da0f4 100644 --- a/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py +++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_to_postgres.py @@ -49,7 +49,7 @@ ) from airflow.providers.google.cloud.transfers.bigquery_to_postgres import BigQueryToPostgresOperator from airflow.providers.ssh.operators.ssh import SSHOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py b/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py index ac305c6d10c99..cb31a3b4d091d 100644 --- a/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py +++ b/tests/system/providers/google/cloud/cloud_build/example_cloud_build.py @@ -38,7 +38,7 @@ CloudBuildListBuildsOperator, CloudBuildRetryBuildOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py index 554d06253e0f1..4884122751e09 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py +++ b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_memcached.py @@ -41,7 +41,7 @@ CloudMemorystoreMemcachedUpdateInstanceOperator, CloudMemorystoreMemcachedUpdateParametersOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py index 971285ec64907..c46d966371dac 100644 --- a/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py +++ b/tests/system/providers/google/cloud/cloud_memorystore/example_cloud_memorystore_redis.py @@ -45,7 +45,7 @@ GCSCreateBucketOperator, GCSDeleteBucketOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py b/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py index ceaf4ed82861f..aebb1e3e7ed85 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py +++ b/tests/system/providers/google/cloud/gcs/example_gcs_copy_delete.py @@ -34,7 +34,7 @@ GCSListObjectsOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py index d1e12e3ee9988..55bec85a50562 100644 --- a/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py @@ -38,7 +38,7 @@ ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py index 7c776edd7f446..a673ab88f722e 100644 --- a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py @@ -46,7 +46,7 @@ GCSDeleteBucketOperator, ) from airflow.providers.ssh.operators.ssh import SSHOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py b/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py index 21b997f821704..2860d8552e101 100644 --- a/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py +++ b/tests/system/providers/google/cloud/gcs/example_sftp_to_gcs.py @@ -29,7 +29,7 @@ from airflow.models.dag import DAG from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.transfers.sftp_to_gcs import SFTPToGCSOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/gcs/example_sheets.py b/tests/system/providers/google/cloud/gcs/example_sheets.py index bd831da4c4157..2247819494f91 100644 --- a/tests/system/providers/google/cloud/gcs/example_sheets.py +++ b/tests/system/providers/google/cloud/gcs/example_sheets.py @@ -30,7 +30,7 @@ from airflow.providers.google.cloud.transfers.sheets_to_gcs import GoogleSheetsToGCSOperator from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator from airflow.providers.google.suite.transfers.gcs_to_sheets import GCSToGoogleSheetsOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py index b58fb9ae60937..173fddad3a065 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py @@ -30,7 +30,7 @@ GKEDeleteClusterOperator, GKEStartPodOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py index 5462700745421..e974a628c7a52 100644 --- a/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py +++ b/tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py @@ -30,7 +30,7 @@ GKEDeleteClusterOperator, GKEStartPodOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py index 48af97834359f..bde2c0bbaf9ee 100644 --- a/tests/system/providers/google/cloud/ml_engine/example_mlengine.py +++ b/tests/system/providers/google/cloud/ml_engine/example_mlengine.py @@ -52,7 +52,7 @@ ListModelVersionsOperator, SetDefaultVersionOnModelOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") diff --git a/tests/system/providers/google/cloud/natural_language/example_natural_language.py b/tests/system/providers/google/cloud/natural_language/example_natural_language.py index aecdba4f7ef68..e04fdf4fb601b 100644 --- a/tests/system/providers/google/cloud/natural_language/example_natural_language.py +++ b/tests/system/providers/google/cloud/natural_language/example_natural_language.py @@ -33,7 +33,7 @@ CloudNaturalLanguageAnalyzeSentimentOperator, CloudNaturalLanguageClassifyTextOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "gcp_natural_language" diff --git a/tests/system/providers/google/cloud/pubsub/example_pubsub.py b/tests/system/providers/google/cloud/pubsub/example_pubsub.py index 157aef2e4fbfa..4ff3091e5fa53 100644 --- a/tests/system/providers/google/cloud/pubsub/example_pubsub.py +++ b/tests/system/providers/google/cloud/pubsub/example_pubsub.py @@ -34,7 +34,7 @@ PubSubPullOperator, ) from airflow.providers.google.cloud.sensors.pubsub import PubSubPullSensor -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py index 58d98cc5c0780..11231c0dfd40c 100644 --- a/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py +++ b/tests/system/providers/google/cloud/sql_to_sheets/example_sql_to_sheets.py @@ -44,7 +44,7 @@ from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator from airflow.providers.google.suite.transfers.sql_to_sheets import SQLToGoogleSheetsOperator from airflow.providers.ssh.operators.ssh import SSHOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session, json from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/tasks/example_queue.py b/tests/system/providers/google/cloud/tasks/example_queue.py index f7e6436d91f9a..4c29b584f5bfc 100644 --- a/tests/system/providers/google/cloud/tasks/example_queue.py +++ b/tests/system/providers/google/cloud/tasks/example_queue.py @@ -45,7 +45,7 @@ CloudTasksQueuesListOperator, CloudTasksQueueUpdateOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py index 5f8086d3dc626..33a289c1ffa1c 100644 --- a/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py +++ b/tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py @@ -46,7 +46,7 @@ ) from airflow.providers.google.cloud.transfers.postgres_to_gcs import PostgresToGCSOperator from airflow.providers.ssh.operators.ssh import SSHOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.settings import Session from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/system/providers/google/cloud/translate/example_translate.py b/tests/system/providers/google/cloud/translate/example_translate.py index 27b653cbf8359..b593060f6e5bb 100644 --- a/tests/system/providers/google/cloud/translate/example_translate.py +++ b/tests/system/providers/google/cloud/translate/example_translate.py @@ -26,7 +26,7 @@ from airflow.models.dag import DAG from airflow.providers.google.cloud.operators.translate import CloudTranslateTextOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator DAG_ID = "gcp_translate" diff --git a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py index f56b1f47beb29..499db2d6427ba 100644 --- a/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py +++ b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py @@ -40,7 +40,7 @@ CloudVideoIntelligenceDetectVideoShotsOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") diff --git a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py b/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py index fb356e8c254fa..2a4d7b75f1332 100644 --- a/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py +++ b/tests/system/providers/google/cloud/vision/example_vision_annotate_image.py @@ -31,7 +31,7 @@ CloudVisionTextDetectOperator, ) from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule # [START howto_operator_vision_retry_import] diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py b/tests/system/providers/google/datacatalog/example_datacatalog_entries.py index 7947994836d3d..47edfb96368f1 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_entries.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_entries.py @@ -35,7 +35,7 @@ CloudDataCatalogUpdateEntryOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py b/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py index 96951f0b78a38..781d047c53469 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_search_catalog.py @@ -37,7 +37,7 @@ CloudDataCatalogSearchCatalogOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py b/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py index 69d91e39ee076..b8dd9170c3c02 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_tag_templates.py @@ -34,7 +34,7 @@ CloudDataCatalogUpdateTagTemplateFieldOperator, CloudDataCatalogUpdateTagTemplateOperator, ) -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py b/tests/system/providers/google/datacatalog/example_datacatalog_tags.py index cc841ddf67d92..17397fcea2806 100644 --- a/tests/system/providers/google/datacatalog/example_datacatalog_tags.py +++ b/tests/system/providers/google/datacatalog/example_datacatalog_tags.py @@ -38,7 +38,7 @@ CloudDataCatalogUpdateTagOperator, ) from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.utils.trigger_rule import TriggerRule from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID diff --git a/tests/system/providers/opsgenie/example_opsgenie_notifier.py b/tests/system/providers/opsgenie/example_opsgenie_notifier.py index e4205bc50768a..10edf8debdaed 100644 --- a/tests/system/providers/opsgenie/example_opsgenie_notifier.py +++ b/tests/system/providers/opsgenie/example_opsgenie_notifier.py @@ -22,7 +22,7 @@ from airflow import DAG from airflow.providers.opsgenie.notifications.opsgenie import send_opsgenie_notification -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator with DAG( "opsgenie_notifier", diff --git a/tests/system/providers/singularity/example_singularity.py b/tests/system/providers/singularity/example_singularity.py index 74f4346e493cb..4b60c080dcd2c 100644 --- a/tests/system/providers/singularity/example_singularity.py +++ b/tests/system/providers/singularity/example_singularity.py @@ -22,7 +22,7 @@ from airflow import DAG from airflow.providers.singularity.operators.singularity import SingularityOperator -from airflow.providers.standard.core.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") DAG_ID = "singularity_sample" diff --git a/tests/test_utils/compat.py b/tests/test_utils/compat.py index 4c49eaecdb677..ce9c8c94bc3b6 100644 --- a/tests/test_utils/compat.py +++ b/tests/test_utils/compat.py @@ -18,9 +18,7 @@ import contextlib import json -import os -from importlib.metadata import version -from typing import TYPE_CHECKING, Any, cast +from typing import Any, cast from packaging.version import Version @@ -53,9 +51,9 @@ from airflow.models.baseoperator import BaseOperatorLink try: - from airflow.providers.standard.core.operators.bash import BashOperator - from airflow.providers.standard.core.sensors.bash import BashSensor - from airflow.providers.standard.time.sensors.date_time import DateTimeSensor + from airflow.providers.standard.operators.bash import BashOperator + from airflow.providers.standard.sensors.bash import BashSensor + from airflow.providers.standard.sensors.date_time import DateTimeSensor except ImportError: # Compatibility for Airflow < 2.10.* from airflow.operators.bash import BashOperator # type: ignore[no-redef,attr-defined] From d2c1c39a7e5af092a4475f096af3335a80f3c695 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 22 Sep 2024 02:05:48 +0100 Subject: [PATCH 32/48] remove core folder from standard provider tests --- tests/providers/standard/core/__init__.py | 16 ---------------- .../standard/core/operators/__init__.py | 16 ---------------- .../providers/standard/core/sensors/__init__.py | 16 ---------------- .../standard/{core => }/operators/test_bash.py | 0 .../standard/{core => }/sensors/test_bash.py | 0 5 files changed, 48 deletions(-) delete mode 100644 tests/providers/standard/core/__init__.py delete mode 100644 tests/providers/standard/core/operators/__init__.py delete mode 100644 tests/providers/standard/core/sensors/__init__.py rename tests/providers/standard/{core => }/operators/test_bash.py (100%) rename tests/providers/standard/{core => }/sensors/test_bash.py (100%) diff --git a/tests/providers/standard/core/__init__.py b/tests/providers/standard/core/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/providers/standard/core/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/standard/core/operators/__init__.py b/tests/providers/standard/core/operators/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/providers/standard/core/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/standard/core/sensors/__init__.py b/tests/providers/standard/core/sensors/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/tests/providers/standard/core/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/standard/core/operators/test_bash.py b/tests/providers/standard/operators/test_bash.py similarity index 100% rename from tests/providers/standard/core/operators/test_bash.py rename to tests/providers/standard/operators/test_bash.py diff --git a/tests/providers/standard/core/sensors/test_bash.py b/tests/providers/standard/sensors/test_bash.py similarity index 100% rename from tests/providers/standard/core/sensors/test_bash.py rename to tests/providers/standard/sensors/test_bash.py From 167fc72af213f2f64eadfab5b629597b4f8c3cdc Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 22 Sep 2024 12:32:04 +0100 Subject: [PATCH 33/48] updating pre-commit hook to allow standard provider imports in core example dags --- .pre-commit-config.yaml | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 951d08f8ab095..ce557dba431b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -550,29 +550,9 @@ repos: - id: check-no-providers-in-core-examples language: pygrep name: No providers imports in core example DAGs - description: The core example DAGs have no dependencies other than core Airflow - entry: "^\\s*from airflow\\.providers.*" + description: The core example DAGs have no dependencies other than standard provider or core Airflow + entry: "^\\s*from airflow\\.providers.(?!standard.)" pass_filenames: true - exclude: > - (?x) - ^airflow/example_dags/example_branch_datetime_operator.py| - ^airflow/example_dags/example_branch_day_of_week_operator.py| - ^airflow/example_dags/example_sensors.py| - ^airflow/example_dags/example_sensors.py| - ^airflow/example_dags/example_sensors.py| - ^airflow/example_dags/example_time_delta_sensor_async.py| - ^airflow/example_dags/example_bash_operator.py| - ^airflow/example_dags/example_complex.py| - ^airflow/example_dags/example_datasets.py| - ^airflow/example_dags/example_inlet_event_extra.py| - ^airflow/example_dags/example_outlet_event_extra.py| - ^airflow/example_dags/example_passing_params_via_test_command.py| - ^airflow/example_dags/example_setup_teardown.py| - ^airflow/example_dags/example_task_group.py| - ^airflow/example_dags/example_trigger_target_dag.py| - ^airflow/example_dags/example_xcom.py| - ^airflow/example_dags/example_xcomargs.py| - ^airflow/example_dags/tutorial.py files: ^airflow/example_dags/.*\.py$ - id: check-no-airflow-deprecation-in-providers language: pygrep From 2c9bfcea5bd732aef4b30c5ddbd95e0865a88dba Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 23 Sep 2024 14:14:02 +0100 Subject: [PATCH 34/48] adding standard provider to hatch_build pre_installed_provider --- hatch_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hatch_build.py b/hatch_build.py index 661bc35b13d13..72f59c79ef9ff 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -49,6 +49,7 @@ "imap", "smtp", "sqlite", + "standard", ] # Those extras are dynamically added by hatch in the build hook to metadata optional dependencies From ea25162e7fffc55dfa2c50ba545d3af4f457c332 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Tue, 24 Sep 2024 21:50:48 +0100 Subject: [PATCH 35/48] skipping k8s tests and adding empty operator example to use temporary test for docker_tests --- .../example_dags/example_empty_operator.py | 43 +++++++++++++++++++ .../test_docker_compose_quick_start.py | 3 +- kubernetes_tests/test_base.py | 2 + kubernetes_tests/test_kubernetes_executor.py | 10 ++++- kubernetes_tests/test_other_executors.py | 10 ++++- prod_image_installed_providers.txt | 1 - 6 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 airflow/example_dags/example_empty_operator.py diff --git a/airflow/example_dags/example_empty_operator.py b/airflow/example_dags/example_empty_operator.py new file mode 100644 index 0000000000000..c7691db754659 --- /dev/null +++ b/airflow/example_dags/example_empty_operator.py @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Empty operator +""" + +from __future__ import annotations + +# [START example] +import datetime + +import pendulum + +from airflow.models.dag import DAG +from airflow.operators.empty import EmptyOperator + +with DAG( + dag_id="example_empty_operator", + schedule="0 0 * * *", + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + catchup=False, + dagrun_timeout=datetime.timedelta(minutes=60), + tags=["example3"], +) as dag: + task1 = EmptyOperator(task_id="task1") + task2 = EmptyOperator(task_id="task2") + task1 >> task2 +# [END example] diff --git a/docker_tests/test_docker_compose_quick_start.py b/docker_tests/test_docker_compose_quick_start.py index eab0a7a62e019..fc03ebe79ce8a 100644 --- a/docker_tests/test_docker_compose_quick_start.py +++ b/docker_tests/test_docker_compose_quick_start.py @@ -31,12 +31,13 @@ # isort:off (needed to workaround isort bug) from docker_tests.command_utils import run_command from docker_tests.constants import SOURCE_ROOT +from docker_tests.test_prod_image import REGULAR_IMAGE_PROVIDERS # isort:on (needed to workaround isort bug) AIRFLOW_WWW_USER_USERNAME = os.environ.get("_AIRFLOW_WWW_USER_USERNAME", "airflow") AIRFLOW_WWW_USER_PASSWORD = os.environ.get("_AIRFLOW_WWW_USER_PASSWORD", "airflow") -DAG_ID = "example_bash_operator" +DAG_ID = "example_bash_operator" if "standard" in REGULAR_IMAGE_PROVIDERS else "example_empty_operator" DAG_RUN_ID = "test_dag_run_id" diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py index 0f5b673db2f93..2a17803af5da9 100644 --- a/kubernetes_tests/test_base.py +++ b/kubernetes_tests/test_base.py @@ -34,6 +34,8 @@ CLUSTER_FORWARDED_PORT = os.environ.get("CLUSTER_FORWARDED_PORT") or "8080" KUBERNETES_HOST_PORT = (os.environ.get("CLUSTER_HOST") or "localhost") + ":" + CLUSTER_FORWARDED_PORT EXECUTOR = os.environ.get("EXECUTOR") +SOURCE_ROOT = Path(__file__).resolve().parents[1] +PROD_IMAGE_PROVIDERS = (SOURCE_ROOT / "prod_image_installed_providers.txt").read_text().splitlines() print() print(f"Cluster host/port used: ${KUBERNETES_HOST_PORT}") diff --git a/kubernetes_tests/test_kubernetes_executor.py b/kubernetes_tests/test_kubernetes_executor.py index a270243bfac78..6ad1b9b7271ab 100644 --- a/kubernetes_tests/test_kubernetes_executor.py +++ b/kubernetes_tests/test_kubernetes_executor.py @@ -20,9 +20,17 @@ import pytest -from kubernetes_tests.test_base import EXECUTOR, BaseK8STest # isort:skip (needed to workaround isort bug) +from kubernetes_tests.test_base import ( + EXECUTOR, + PROD_IMAGE_PROVIDERS, # isort:skip (needed to workaround isort bug) + BaseK8STest, +) +@pytest.mark.skipif( + "standard" not in PROD_IMAGE_PROVIDERS, + reason="Skipping temporarily due to standard provider not available", +) @pytest.mark.skipif(EXECUTOR != "KubernetesExecutor", reason="Only runs on KubernetesExecutor") class TestKubernetesExecutor(BaseK8STest): @pytest.mark.execution_timeout(300) diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py index 97b7e3df728ee..b316885e9d07f 100644 --- a/kubernetes_tests/test_other_executors.py +++ b/kubernetes_tests/test_other_executors.py @@ -20,12 +20,20 @@ import pytest -from kubernetes_tests.test_base import EXECUTOR, BaseK8STest # isort:skip (needed to workaround isort bug) +from kubernetes_tests.test_base import ( + EXECUTOR, + PROD_IMAGE_PROVIDERS, # isort:skip (needed to workaround isort bug) + BaseK8STest, +) # These tests are here because only KubernetesExecutor can run the tests in # test_kubernetes_executor.py # Also, the skipping is necessary as there's no gain in running these tests in KubernetesExecutor +@pytest.mark.skipif( + "standard" not in PROD_IMAGE_PROVIDERS, + reason="Skipping temporarily due to standard provider not available", +) @pytest.mark.skipif(EXECUTOR == "KubernetesExecutor", reason="Does not run on KubernetesExecutor") class TestCeleryAndLocalExecutor(BaseK8STest): def test_integration_run_dag(self): diff --git a/prod_image_installed_providers.txt b/prod_image_installed_providers.txt index 5ba30a865f4f3..7340928738c11 100644 --- a/prod_image_installed_providers.txt +++ b/prod_image_installed_providers.txt @@ -27,4 +27,3 @@ smtp snowflake sqlite ssh -standard From f9aa11fcfc71444c4dd123b22824a1ecf2e2d2f3 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Tue, 24 Sep 2024 23:17:07 +0100 Subject: [PATCH 36/48] skipping docker test test_trigger_dag_and_wait_for_result temporarily --- .../example_dags/example_empty_operator.py | 43 ------------------- .../celery/executors/celery_executor_utils.py | 9 +++- .../test_docker_compose_quick_start.py | 6 ++- hatch_build.py | 1 - 4 files changed, 13 insertions(+), 46 deletions(-) delete mode 100644 airflow/example_dags/example_empty_operator.py diff --git a/airflow/example_dags/example_empty_operator.py b/airflow/example_dags/example_empty_operator.py deleted file mode 100644 index c7691db754659..0000000000000 --- a/airflow/example_dags/example_empty_operator.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Example Empty operator -""" - -from __future__ import annotations - -# [START example] -import datetime - -import pendulum - -from airflow.models.dag import DAG -from airflow.operators.empty import EmptyOperator - -with DAG( - dag_id="example_empty_operator", - schedule="0 0 * * *", - start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), - catchup=False, - dagrun_timeout=datetime.timedelta(minutes=60), - tags=["example3"], -) as dag: - task1 = EmptyOperator(task_id="task1") - task2 = EmptyOperator(task_id="task2") - task1 >> task2 -# [END example] diff --git a/airflow/providers/celery/executors/celery_executor_utils.py b/airflow/providers/celery/executors/celery_executor_utils.py index 58b11169b9be2..a7aa6a87ea2cf 100644 --- a/airflow/providers/celery/executors/celery_executor_utils.py +++ b/airflow/providers/celery/executors/celery_executor_utils.py @@ -109,7 +109,14 @@ def on_celery_import_modules(*args, **kwargs): """ import jinja2.ext # noqa: F401 - import airflow.providers.standard.operators.bash # noqa: F401 + import airflow.jobs.local_task_job_runner + import airflow.macros + import airflow.operators.python + + try: + import airflow.providers.standard.operators.bash + except ImportError: + import airflow.operators.bash # noqa: F401 with contextlib.suppress(ImportError): import numpy # noqa: F401 diff --git a/docker_tests/test_docker_compose_quick_start.py b/docker_tests/test_docker_compose_quick_start.py index fc03ebe79ce8a..4534a78a02e4f 100644 --- a/docker_tests/test_docker_compose_quick_start.py +++ b/docker_tests/test_docker_compose_quick_start.py @@ -37,7 +37,7 @@ AIRFLOW_WWW_USER_USERNAME = os.environ.get("_AIRFLOW_WWW_USER_USERNAME", "airflow") AIRFLOW_WWW_USER_PASSWORD = os.environ.get("_AIRFLOW_WWW_USER_PASSWORD", "airflow") -DAG_ID = "example_bash_operator" if "standard" in REGULAR_IMAGE_PROVIDERS else "example_empty_operator" +DAG_ID = "example_bash_operator" DAG_RUN_ID = "test_dag_run_id" @@ -66,6 +66,10 @@ def wait_for_terminal_dag_state(dag_id, dag_run_id): break +@pytest.mark.skipif( + "standard" not in REGULAR_IMAGE_PROVIDERS, + reason="Skipping temporarily due to standard provider not available.", +) def test_trigger_dag_and_wait_for_result(default_docker_image, tmp_path_factory, monkeypatch): """Simple test which reproduce setup docker-compose environment and trigger example dag.""" tmp_dir = tmp_path_factory.mktemp("airflow-quick-start") diff --git a/hatch_build.py b/hatch_build.py index 72f59c79ef9ff..661bc35b13d13 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -49,7 +49,6 @@ "imap", "smtp", "sqlite", - "standard", ] # Those extras are dynamically added by hatch in the build hook to metadata optional dependencies From 70a3bf24e587c939c48d80d810a391a3c72c22bd Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 3 Oct 2024 08:40:41 +0100 Subject: [PATCH 37/48] remove skipif from k8s and docker tests --- docker_tests/test_docker_compose_quick_start.py | 5 ----- kubernetes_tests/test_base.py | 2 -- kubernetes_tests/test_kubernetes_executor.py | 7 +------ kubernetes_tests/test_other_executors.py | 7 +------ 4 files changed, 2 insertions(+), 19 deletions(-) diff --git a/docker_tests/test_docker_compose_quick_start.py b/docker_tests/test_docker_compose_quick_start.py index 4534a78a02e4f..eab0a7a62e019 100644 --- a/docker_tests/test_docker_compose_quick_start.py +++ b/docker_tests/test_docker_compose_quick_start.py @@ -31,7 +31,6 @@ # isort:off (needed to workaround isort bug) from docker_tests.command_utils import run_command from docker_tests.constants import SOURCE_ROOT -from docker_tests.test_prod_image import REGULAR_IMAGE_PROVIDERS # isort:on (needed to workaround isort bug) @@ -66,10 +65,6 @@ def wait_for_terminal_dag_state(dag_id, dag_run_id): break -@pytest.mark.skipif( - "standard" not in REGULAR_IMAGE_PROVIDERS, - reason="Skipping temporarily due to standard provider not available.", -) def test_trigger_dag_and_wait_for_result(default_docker_image, tmp_path_factory, monkeypatch): """Simple test which reproduce setup docker-compose environment and trigger example dag.""" tmp_dir = tmp_path_factory.mktemp("airflow-quick-start") diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py index 2a17803af5da9..0f5b673db2f93 100644 --- a/kubernetes_tests/test_base.py +++ b/kubernetes_tests/test_base.py @@ -34,8 +34,6 @@ CLUSTER_FORWARDED_PORT = os.environ.get("CLUSTER_FORWARDED_PORT") or "8080" KUBERNETES_HOST_PORT = (os.environ.get("CLUSTER_HOST") or "localhost") + ":" + CLUSTER_FORWARDED_PORT EXECUTOR = os.environ.get("EXECUTOR") -SOURCE_ROOT = Path(__file__).resolve().parents[1] -PROD_IMAGE_PROVIDERS = (SOURCE_ROOT / "prod_image_installed_providers.txt").read_text().splitlines() print() print(f"Cluster host/port used: ${KUBERNETES_HOST_PORT}") diff --git a/kubernetes_tests/test_kubernetes_executor.py b/kubernetes_tests/test_kubernetes_executor.py index 6ad1b9b7271ab..42b181b443041 100644 --- a/kubernetes_tests/test_kubernetes_executor.py +++ b/kubernetes_tests/test_kubernetes_executor.py @@ -22,15 +22,10 @@ from kubernetes_tests.test_base import ( EXECUTOR, - PROD_IMAGE_PROVIDERS, # isort:skip (needed to workaround isort bug) - BaseK8STest, + BaseK8STest, # isort:skip (needed to workaround isort bug) ) -@pytest.mark.skipif( - "standard" not in PROD_IMAGE_PROVIDERS, - reason="Skipping temporarily due to standard provider not available", -) @pytest.mark.skipif(EXECUTOR != "KubernetesExecutor", reason="Only runs on KubernetesExecutor") class TestKubernetesExecutor(BaseK8STest): @pytest.mark.execution_timeout(300) diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py index b316885e9d07f..3a1aea16f1291 100644 --- a/kubernetes_tests/test_other_executors.py +++ b/kubernetes_tests/test_other_executors.py @@ -22,18 +22,13 @@ from kubernetes_tests.test_base import ( EXECUTOR, - PROD_IMAGE_PROVIDERS, # isort:skip (needed to workaround isort bug) - BaseK8STest, + BaseK8STest, # isort:skip (needed to workaround isort bug) ) # These tests are here because only KubernetesExecutor can run the tests in # test_kubernetes_executor.py # Also, the skipping is necessary as there's no gain in running these tests in KubernetesExecutor -@pytest.mark.skipif( - "standard" not in PROD_IMAGE_PROVIDERS, - reason="Skipping temporarily due to standard provider not available", -) @pytest.mark.skipif(EXECUTOR == "KubernetesExecutor", reason="Does not run on KubernetesExecutor") class TestCeleryAndLocalExecutor(BaseK8STest): def test_integration_run_dag(self): From e06b84f9b974a53c97f419fe1cf7123b3d920b37 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 3 Oct 2024 09:04:13 +0100 Subject: [PATCH 38/48] update missing provider in hatch_build --- hatch_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hatch_build.py b/hatch_build.py index 661bc35b13d13..72f59c79ef9ff 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -49,6 +49,7 @@ "imap", "smtp", "sqlite", + "standard", ] # Those extras are dynamically added by hatch in the build hook to metadata optional dependencies From 46f7f4a4ef1893bdd2290983c63079ed89505c84 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 3 Oct 2024 09:25:07 +0100 Subject: [PATCH 39/48] fix rebase issues: adding typechecking --- tests/test_utils/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/compat.py b/tests/test_utils/compat.py index ce9c8c94bc3b6..67a111350daff 100644 --- a/tests/test_utils/compat.py +++ b/tests/test_utils/compat.py @@ -18,7 +18,7 @@ import contextlib import json -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from packaging.version import Version From d2328ec0b286d7244b88521a6bb8f6a70cc4d59f Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 3 Oct 2024 09:51:20 +0100 Subject: [PATCH 40/48] fix rebase issues: remove extra imports from test_views_tasks --- tests/www/views/test_views_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 85ce160aca8f5..4dcb7252a3658 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -43,7 +43,6 @@ from airflow.utils.state import DagRunState, State from airflow.utils.types import DagRunType from airflow.www.views import TaskInstanceModelView, _safe_parse_datetime -from tests.test_utils.api_connexion_utils import create_user, delete_roles, delete_user from tests.providers.fab.auth_manager.api_endpoints.api_connexion_utils import ( create_user, delete_roles, From 1912463ef4a33529bd40eae7b0cad3796bf6a239 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 3 Oct 2024 10:15:43 +0100 Subject: [PATCH 41/48] fix rebase issues: remove example_datasets --- airflow/example_dags/example_datasets.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 airflow/example_dags/example_datasets.py diff --git a/airflow/example_dags/example_datasets.py b/airflow/example_dags/example_datasets.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 9f99690b6a14abddfa7d6058acae096682e88ea7 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Thu, 3 Oct 2024 14:53:40 +0100 Subject: [PATCH 42/48] fix imports in example_assets --- airflow/example_dags/example_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/example_dags/example_assets.py b/airflow/example_dags/example_assets.py index 66369794ed999..451f17a3a3abd 100644 --- a/airflow/example_dags/example_assets.py +++ b/airflow/example_dags/example_assets.py @@ -56,7 +56,7 @@ from airflow.assets import Asset from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator +from airflow.providers.standard.operators.bash import BashOperator from airflow.timetables.assets import AssetOrTimeSchedule from airflow.timetables.trigger import CronTriggerTimetable From 0372b0329b3d932a48e931d6c71ef0f7c01a61d0 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 4 Oct 2024 15:45:19 +0100 Subject: [PATCH 43/48] update standard version to PRE_INSTALLED_PROVIDERS --- hatch_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 72f59c79ef9ff..26ba7e9ceb490 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -49,7 +49,7 @@ "imap", "smtp", "sqlite", - "standard", + "standard>=1.0.0.dev0", ] # Those extras are dynamically added by hatch in the build hook to metadata optional dependencies From 2797a589d964865c02cca70f081243d1a5cb6712 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 4 Oct 2024 19:05:20 +0100 Subject: [PATCH 44/48] updating prod-image-build file to install standard provider inside container --- .github/workflows/prod-image-build.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 75d9d0054ec78..1b85e1bab50a3 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -163,6 +163,15 @@ jobs: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true' && inputs.build-provider-packages == 'true' + - name: "Prepare standard provider package" + shell: bash + run: > + breeze release-management prepare-provider-packages + standard --package-format wheel --skip-tag-check + if: > + inputs.do-build == 'true' && + inputs.upload-package-artifact == 'true' && + inputs.build-provider-packages == 'true' - name: "Prepare chicken-eggs provider packages" shell: bash run: > From f6c6bf87cd218798deea464377def475511ac723 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 4 Oct 2024 19:37:32 +0100 Subject: [PATCH 45/48] update standard provider in prod_image_installed_providers --- .github/workflows/prod-image-build.yml | 8 ++------ prod_image_installed_providers.txt | 1 + 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 1b85e1bab50a3..55754d48877a1 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -166,12 +166,8 @@ jobs: - name: "Prepare standard provider package" shell: bash run: > - breeze release-management prepare-provider-packages - standard --package-format wheel --skip-tag-check - if: > - inputs.do-build == 'true' && - inputs.upload-package-artifact == 'true' && - inputs.build-provider-packages == 'true' + breeze release-management prepare-provider-packages standard --package-format wheel + if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true' - name: "Prepare chicken-eggs provider packages" shell: bash run: > diff --git a/prod_image_installed_providers.txt b/prod_image_installed_providers.txt index 7340928738c11..5ba30a865f4f3 100644 --- a/prod_image_installed_providers.txt +++ b/prod_image_installed_providers.txt @@ -27,3 +27,4 @@ smtp snowflake sqlite ssh +standard From 13d80c295648ed1143a4b04cbcbd2011756487d6 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 4 Oct 2024 20:28:13 +0100 Subject: [PATCH 46/48] adding standard provider to CHICKEN_EGG_PROVIDERS in global constants --- .github/workflows/prod-image-build.yml | 5 ----- prod_image_installed_providers.txt | 1 - 2 files changed, 6 deletions(-) diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 55754d48877a1..75d9d0054ec78 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -163,11 +163,6 @@ jobs: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true' && inputs.build-provider-packages == 'true' - - name: "Prepare standard provider package" - shell: bash - run: > - breeze release-management prepare-provider-packages standard --package-format wheel - if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true' - name: "Prepare chicken-eggs provider packages" shell: bash run: > diff --git a/prod_image_installed_providers.txt b/prod_image_installed_providers.txt index 5ba30a865f4f3..7340928738c11 100644 --- a/prod_image_installed_providers.txt +++ b/prod_image_installed_providers.txt @@ -27,4 +27,3 @@ smtp snowflake sqlite ssh -standard From 76442b73aca6d9d53cca655c7ae014055948165d Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Fri, 4 Oct 2024 21:26:45 +0100 Subject: [PATCH 47/48] remove version from hatch build --- hatch_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 26ba7e9ceb490..72f59c79ef9ff 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -49,7 +49,7 @@ "imap", "smtp", "sqlite", - "standard>=1.0.0.dev0", + "standard", ] # Those extras are dynamically added by hatch in the build hook to metadata optional dependencies From 726518b7515fa1657a33888fc70c0de3484edf6b Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 7 Oct 2024 17:08:02 +0100 Subject: [PATCH 48/48] add missing standard provider in prod_image_installed_providers --- prod_image_installed_providers.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/prod_image_installed_providers.txt b/prod_image_installed_providers.txt index 7340928738c11..5ba30a865f4f3 100644 --- a/prod_image_installed_providers.txt +++ b/prod_image_installed_providers.txt @@ -27,3 +27,4 @@ smtp snowflake sqlite ssh +standard