diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index c853c7de833f8..48a7c71c4a327 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -286,15 +286,7 @@ dev = [ "apache-airflow-ctl", "apache-airflow-devel-common", "apache-airflow-task-sdk", - # TODO(potiuk): eventually we do not want any providers nor apache-airflow extras to be needed for - # airflow-core tests "apache-airflow[pandas,polars]", - "apache-airflow-providers-amazon", - "apache-airflow-providers-celery", - "apache-airflow-providers-cncf-kubernetes", - "apache-airflow-providers-fab>=2.2.0", - "apache-airflow-providers-git", - "apache-airflow-providers-ftp", ] # To build docs: diff --git a/airflow-core/tests/integration/cli/commands/test_celery_command.py b/airflow-core/tests/integration/cli/commands/test_celery_command.py index 20cc664f997e5..6d731d4085a5a 100644 --- a/airflow-core/tests/integration/cli/commands/test_celery_command.py +++ b/airflow-core/tests/integration/cli/commands/test_celery_command.py @@ -24,6 +24,8 @@ from airflow.cli import cli_parser from airflow.executors import executor_loader + +pytest.importorskip("airflow.providers.celery", reason="celery provider required") from airflow.providers.celery.cli import celery_command from tests_common.test_utils.config import conf_vars diff --git a/airflow-core/tests/unit/always/test_example_dags.py b/airflow-core/tests/unit/always/test_example_dags.py index b2fca80e8700d..1fdc8788ca07c 100644 --- a/airflow-core/tests/unit/always/test_example_dags.py +++ b/airflow-core/tests/unit/always/test_example_dags.py @@ -217,6 +217,10 @@ def test_should_be_importable(example: str, patch_get_dagbag_import_timeout): f"Skipping {example} because it requires an optional provider feature that is not installed." ) assert len(dagbag.import_errors) == 0, f"import_errors={str(dagbag.import_errors)}" + if len(dagbag.dag_ids) == 0: + pytest.skip( + f"Skipping {example} because it produced no DAGs (likely due to missing optional dependencies)." + ) assert len(dagbag.dag_ids) >= 1 diff --git a/airflow-core/tests/unit/always/test_providers_manager.py b/airflow-core/tests/unit/always/test_providers_manager.py index 7b69196501291..6f7592dd71c60 100644 --- a/airflow-core/tests/unit/always/test_providers_manager.py +++ b/airflow-core/tests/unit/always/test_providers_manager.py @@ -41,6 +41,15 @@ from tests_common.test_utils.markers import skip_if_force_lowest_dependencies_marker + +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + if TYPE_CHECKING: from unittest.mock import MagicMock @@ -416,6 +425,9 @@ def test_add_customized_fields(self): assert behaviour["relabeling"] == {"login": "Email Address"} assert behaviour["placeholders"]["host"] == "smtp.gmail.com" + @pytest.mark.skipif( + not _provider_installed("airflow.providers.http"), reason="http provider not installed" + ) def test_load_ui_for_http_provider(self): """Test that HTTP provider ui metadata is loaded from provider info.""" pm = ProvidersManager() @@ -428,6 +440,9 @@ def test_load_ui_for_http_provider(self): assert "relabeling" in behaviour assert "placeholders" in behaviour + @pytest.mark.skipif( + not _provider_installed("airflow.providers.http"), reason="http provider not installed" + ) def test_ui_metadata_loading_without_hook_import(self): """Test that UI metadata loads from provider info without importing hook classes.""" with patch("airflow.providers_manager.import_string") as mock_import: diff --git a/airflow-core/tests/unit/api_fastapi/conftest.py b/airflow-core/tests/unit/api_fastapi/conftest.py index aace17f8a1479..9ff55a668f2dd 100644 --- a/airflow-core/tests/unit/api_fastapi/conftest.py +++ b/airflow-core/tests/unit/api_fastapi/conftest.py @@ -29,7 +29,6 @@ from airflow.api_fastapi.auth.managers.simple.user import SimpleAuthManagerUser from airflow.dag_processing.bundles.manager import DagBundlesManager from airflow.models import Connection -from airflow.providers.git.bundles.git import GitDagBundle from airflow.providers.standard.operators.empty import EmptyOperator from tests_common.test_utils.config import conf_vars @@ -127,6 +126,9 @@ def create_test_client(apps="all"): @pytest.fixture def configure_git_connection_for_dag_bundle(session): + pytest.importorskip("airflow.providers.git.bundles.git") + from airflow.providers.git.bundles.git import GitDagBundle + clear_db_connections(False) # Git connection is required for the bundles to have a url. connection = Connection( diff --git a/airflow-core/tests/unit/cli/commands/test_config_command.py b/airflow-core/tests/unit/cli/commands/test_config_command.py index 008554422624b..4cc21e550deaa 100644 --- a/airflow-core/tests/unit/cli/commands/test_config_command.py +++ b/airflow-core/tests/unit/cli/commands/test_config_command.py @@ -30,6 +30,15 @@ from tests_common.test_utils.config import conf_vars + +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + STATSD_CONFIG_BEGIN_WITH = "# `StatsD `" @@ -220,6 +229,10 @@ def test_cli_show_changed_defaults_when_overridden_in_env(self, stdout_capture): lines = output.splitlines() assert any(line.startswith("hostname_callable = test_env") for line in lines if line) + @pytest.mark.skipif( + not _provider_installed("airflow.providers.celery"), + reason="celery provider not installed", + ) def test_cli_has_providers(self, stdout_capture): with stdout_capture as temp_stdout: config_command.show_config( diff --git a/airflow-core/tests/unit/cli/commands/test_connection_command.py b/airflow-core/tests/unit/cli/commands/test_connection_command.py index c62dfb4c5b357..0fffa83a58af4 100644 --- a/airflow-core/tests/unit/cli/commands/test_connection_command.py +++ b/airflow-core/tests/unit/cli/commands/test_connection_command.py @@ -1060,6 +1060,7 @@ def setup_class(self): clear_db_connections() def test_cli_connections_test_success(self, mocker, stdout_capture): + pytest.importorskip("airflow.providers.http") mocker.patch.dict(os.environ, {"AIRFLOW__CORE__TEST_CONNECTION": "Enabled"}) mock_test_conn = mocker.patch("airflow.providers.http.hooks.http.HttpHook.test_connection") conn_id = "http_default" @@ -1069,6 +1070,7 @@ def test_cli_connections_test_success(self, mocker, stdout_capture): assert "Connection success!" in stdout.getvalue() def test_cli_connections_test_fail(self, mocker, stdout_capture): + pytest.importorskip("airflow.providers.http") mocker.patch.dict(os.environ, {"AIRFLOW__CORE__TEST_CONNECTION": "Enabled"}) mock_test_conn = mocker.patch("airflow.providers.http.hooks.http.HttpHook.test_connection") conn_id = "http_default" diff --git a/airflow-core/tests/unit/cli/commands/test_dag_command.py b/airflow-core/tests/unit/cli/commands/test_dag_command.py index 208fd9bb074f3..62af3592ca31d 100644 --- a/airflow-core/tests/unit/cli/commands/test_dag_command.py +++ b/airflow-core/tests/unit/cli/commands/test_dag_command.py @@ -55,6 +55,7 @@ clear_db_runs, parse_and_sync_to_db, ) +from tests_common.test_utils.markers import skip_if_force_lowest_dependencies_marker from unit.models import TEST_DAGS_FOLDER DEFAULT_DATE = timezone.make_aware(datetime(2015, 1, 1), timezone=timezone.utc) @@ -491,6 +492,7 @@ def test_pause(self): dag_command.dag_unpause(args) assert not DagModel.get_dagmodel("example_bash_operator").is_paused + @skip_if_force_lowest_dependencies_marker @mock.patch("airflow.cli.commands.dag_command.ask_yesno") def test_pause_regex(self, mock_yesno): args = self.parser.parse_args(["dags", "pause", "^example_.*$", "--treat-dag-id-as-regex"]) @@ -506,6 +508,7 @@ def test_pause_regex(self, mock_yesno): assert not DagModel.get_dagmodel("example_kubernetes_executor").is_paused assert not DagModel.get_dagmodel("example_xcom_args").is_paused + @skip_if_force_lowest_dependencies_marker @mock.patch("airflow.cli.commands.dag_command.ask_yesno") def test_pause_regex_operation_cancelled(self, ask_yesno, capsys): args = self.parser.parse_args(["dags", "pause", "example_bash_operator", "--treat-dag-id-as-regex"]) diff --git a/airflow-core/tests/unit/cli/commands/test_info_command.py b/airflow-core/tests/unit/cli/commands/test_info_command.py index 78b5e2b948f97..97cc0640b5198 100644 --- a/airflow-core/tests/unit/cli/commands/test_info_command.py +++ b/airflow-core/tests/unit/cli/commands/test_info_command.py @@ -34,6 +34,14 @@ from tests_common.test_utils.config import conf_vars +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + def capture_show_output(instance): console = Console() with console.capture() as capture: @@ -90,6 +98,10 @@ def teardown_class(cls) -> None: def unique_items(items): return {i[0] for i in items} + @pytest.mark.skipif( + not _provider_installed("airflow.providers.amazon"), + reason="amazon provider not installed", + ) @conf_vars( { ("core", "executor"): "TEST_EXECUTOR", diff --git a/airflow-core/tests/unit/cli/commands/test_scheduler_command.py b/airflow-core/tests/unit/cli/commands/test_scheduler_command.py index 2c9e995efabcc..1637aacb88105 100644 --- a/airflow-core/tests/unit/cli/commands/test_scheduler_command.py +++ b/airflow-core/tests/unit/cli/commands/test_scheduler_command.py @@ -30,6 +30,15 @@ from tests_common.test_utils.config import conf_vars + +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + pytestmark = pytest.mark.db_test @@ -41,9 +50,23 @@ def setup_class(cls): @pytest.mark.parametrize( ("executor", "expect_serve_logs"), [ - ("CeleryExecutor", False), + pytest.param( + "CeleryExecutor", + False, + marks=pytest.mark.skipif( + not _provider_installed("airflow.providers.celery"), + reason="celery provider not installed", + ), + ), ("LocalExecutor", True), - ("KubernetesExecutor", False), + pytest.param( + "KubernetesExecutor", + False, + marks=pytest.mark.skipif( + not _provider_installed("airflow.providers.cncf.kubernetes"), + reason="cncf.kubernetes provider not installed", + ), + ), ], ) @mock.patch("airflow.cli.commands.scheduler_command.SchedulerJobRunner") diff --git a/airflow-core/tests/unit/cli/commands/test_standalone_command.py b/airflow-core/tests/unit/cli/commands/test_standalone_command.py index b233b20d5120e..a759a2d54033d 100644 --- a/airflow-core/tests/unit/cli/commands/test_standalone_command.py +++ b/airflow-core/tests/unit/cli/commands/test_standalone_command.py @@ -31,10 +31,34 @@ ) +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + class TestStandaloneCommand: @pytest.mark.parametrize( "conf_executor_name", - [LOCAL_EXECUTOR, CELERY_EXECUTOR, KUBERNETES_EXECUTOR], + [ + LOCAL_EXECUTOR, + pytest.param( + CELERY_EXECUTOR, + marks=pytest.mark.skipif( + not _provider_installed("airflow.providers.celery"), + reason="celery provider not installed", + ), + ), + pytest.param( + KUBERNETES_EXECUTOR, + marks=pytest.mark.skipif( + not _provider_installed("airflow.providers.cncf.kubernetes"), + reason="cncf.kubernetes provider not installed", + ), + ), + ], ) def test_calculate_env(self, conf_executor_name): """Should always force a local executor compatible with the db.""" diff --git a/airflow-core/tests/unit/cli/conftest.py b/airflow-core/tests/unit/cli/conftest.py index 9c20c9d7e6a6d..110a6bb6847ef 100644 --- a/airflow-core/tests/unit/cli/conftest.py +++ b/airflow-core/tests/unit/cli/conftest.py @@ -23,8 +23,6 @@ from airflow.dag_processing.dagbag import DagBag from airflow.executors import local_executor -from airflow.providers.celery.executors import celery_executor -from airflow.providers.cncf.kubernetes.executors import kubernetes_executor from tests_common.test_utils.config import conf_vars from tests_common.test_utils.stream_capture_manager import ( @@ -36,15 +34,25 @@ # Create custom executors here because conftest is imported first custom_executor_module = type(sys)("custom_executor") -custom_executor_module.CustomCeleryExecutor = type( # type: ignore - "CustomCeleryExecutor", (celery_executor.CeleryExecutor,), {} -) custom_executor_module.CustomLocalExecutor = type( # type: ignore "CustomLocalExecutor", (local_executor.LocalExecutor,), {} ) -custom_executor_module.CustomKubernetesExecutor = type( # type: ignore - "CustomKubernetesExecutor", (kubernetes_executor.KubernetesExecutor,), {} -) +try: + from airflow.providers.celery.executors import celery_executor + + custom_executor_module.CustomCeleryExecutor = type( # type: ignore + "CustomCeleryExecutor", (celery_executor.CeleryExecutor,), {} + ) +except ImportError: + pass +try: + from airflow.providers.cncf.kubernetes.executors import kubernetes_executor + + custom_executor_module.CustomKubernetesExecutor = type( # type: ignore + "CustomKubernetesExecutor", (kubernetes_executor.KubernetesExecutor,), {} + ) +except ImportError: + pass sys.modules["custom_executor"] = custom_executor_module diff --git a/airflow-core/tests/unit/cli/test_cli_parser.py b/airflow-core/tests/unit/cli/test_cli_parser.py index c4d8a95f008ba..424446e4cb85e 100644 --- a/airflow-core/tests/unit/cli/test_cli_parser.py +++ b/airflow-core/tests/unit/cli/test_cli_parser.py @@ -44,6 +44,18 @@ from tests_common.test_utils.config import conf_vars + +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + +_celery_installed = _provider_installed("airflow.providers.celery") +_kubernetes_installed = _provider_installed("airflow.providers.cncf.kubernetes") + pytestmark = pytest.mark.db_test # Can not be `--snake_case` or contain uppercase letter @@ -533,13 +545,33 @@ def test_variables_import_help_message_consistency(self): @pytest.mark.parametrize( ("executor", "expected_args"), [ - ("CeleryExecutor", ["celery"]), - ("KubernetesExecutor", ["kubernetes"]), + pytest.param( + "CeleryExecutor", + ["celery"], + marks=pytest.mark.skipif(not _celery_installed, reason="celery provider not installed"), + ), + pytest.param( + "KubernetesExecutor", + ["kubernetes"], + marks=pytest.mark.skipif( + not _kubernetes_installed, reason="cncf.kubernetes provider not installed" + ), + ), ("LocalExecutor", []), # custom executors are mapped to the regular ones in `conftest.py` ("custom_executor.CustomLocalExecutor", []), - ("custom_executor.CustomCeleryExecutor", ["celery"]), - ("custom_executor.CustomKubernetesExecutor", ["kubernetes"]), + pytest.param( + "custom_executor.CustomCeleryExecutor", + ["celery"], + marks=pytest.mark.skipif(not _celery_installed, reason="celery provider not installed"), + ), + pytest.param( + "custom_executor.CustomKubernetesExecutor", + ["kubernetes"], + marks=pytest.mark.skipif( + not _kubernetes_installed, reason="cncf.kubernetes provider not installed" + ), + ), ], ) def test_cli_parser_executors(self, executor, expected_args): @@ -637,6 +669,7 @@ def test_cli_run_time(self): # Minimum run time of Airflow CLI should at least be within 5s assert timing_result < threshold + @pytest.mark.skipif(not _celery_installed, reason="celery provider not installed") def test_airflow_config_contains_providers(self): """ Test that airflow config has providers included by default. @@ -655,6 +688,7 @@ def test_airflow_config_contains_providers(self): assert CONFIG_FILE.exists() assert "celery_config_options" in CONFIG_FILE.read_text() + @pytest.mark.skipif(not _celery_installed, reason="celery provider not installed") def test_airflow_config_output_contains_providers_by_default(self): """Test that airflow config has providers excluded in config list when asked for it.""" CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True) diff --git a/airflow-core/tests/unit/core/test_configuration.py b/airflow-core/tests/unit/core/test_configuration.py index be0f191f8c9b7..b21dbab54129f 100644 --- a/airflow-core/tests/unit/core/test_configuration.py +++ b/airflow-core/tests/unit/core/test_configuration.py @@ -55,6 +55,15 @@ use_config, ) + +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + HOME_DIR = os.path.expanduser("~") # The conf has been updated with deactivate_stale_dags_interval to test the @@ -949,6 +958,10 @@ def test_collect_kwarg_env_vars(self): result = _collect_kwarg_env_vars("AIRFLOW__SECRETS__BACKEND_KWARG__") assert result == {"role_id": "abc"} + @pytest.mark.skipif( + not _provider_installed("airflow.providers.amazon"), + reason="amazon provider not installed", + ) @conf_vars( { ( diff --git a/airflow-core/tests/unit/core/test_exceptions.py b/airflow-core/tests/unit/core/test_exceptions.py index b0ebce05e90d6..944cc0cd77181 100644 --- a/airflow-core/tests/unit/core/test_exceptions.py +++ b/airflow-core/tests/unit/core/test_exceptions.py @@ -18,6 +18,8 @@ import sys +import pytest + class TestExceptions: def setup_method(self): @@ -32,6 +34,7 @@ def teardown_method(self): def test_pod_mutation_hook_exceptions_compatibility( self, ): + pytest.importorskip("airflow.providers.cncf.kubernetes") from airflow.exceptions import ( PodMutationHookException as CoreMutationHookException, ) @@ -48,6 +51,7 @@ def test_pod_mutation_hook_exceptions_compatibility( def test_pod_reconciliation_error_exceptions_compatibility( self, ): + pytest.importorskip("airflow.providers.cncf.kubernetes") from airflow.exceptions import ( PodReconciliationError as CoreReconciliationError, ) diff --git a/airflow-core/tests/unit/serialization/test_dag_serialization.py b/airflow-core/tests/unit/serialization/test_dag_serialization.py index 375b13dea3561..cabd1f9889863 100644 --- a/airflow-core/tests/unit/serialization/test_dag_serialization.py +++ b/airflow-core/tests/unit/serialization/test_dag_serialization.py @@ -44,6 +44,8 @@ import pendulum import pytest from dateutil.relativedelta import FR, relativedelta + +pytest.importorskip("kubernetes", reason="kubernetes package required for DAG serialization tests") from kubernetes.client import models as k8s import airflow diff --git a/airflow-core/tests/unit/serialization/test_serialized_objects.py b/airflow-core/tests/unit/serialization/test_serialized_objects.py index 50e1621c4afb8..1780c6c756419 100644 --- a/airflow-core/tests/unit/serialization/test_serialized_objects.py +++ b/airflow-core/tests/unit/serialization/test_serialized_objects.py @@ -27,6 +27,8 @@ import pendulum import pytest from dateutil import relativedelta + +pytest.importorskip("kubernetes", reason="kubernetes package required for serialized objects tests") from kubernetes.client import models as k8s from pendulum.tz.timezone import FixedTimezone, Timezone from uuid6 import uuid7 diff --git a/airflow-core/tests/unit/utils/test_db.py b/airflow-core/tests/unit/utils/test_db.py index d2f13e27e9e70..75c1b97835691 100644 --- a/airflow-core/tests/unit/utils/test_db.py +++ b/airflow-core/tests/unit/utils/test_db.py @@ -53,6 +53,15 @@ from tests_common.test_utils.config import conf_vars + +def _provider_installed(module_path: str) -> bool: + try: + __import__(module_path) + return True + except ImportError: + return False + + pytestmark = pytest.mark.db_test @@ -125,6 +134,7 @@ def test_upgradedb_uses_migrations_for_empty_db_when_flag_enabled(self, mocker): mock_initdb.assert_not_called() def test_database_schema_and_sqlalchemy_model_are_in_sync(self, initialized_db): + pytest.importorskip("airflow.providers.fab") import airflow.models # Ensure we have a fresh connection for schema comparison @@ -247,7 +257,7 @@ def test_check_migrations(self): }, 1, ), - ( + pytest.param( { ( "core", @@ -255,6 +265,10 @@ def test_check_migrations(self): ): "airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager" }, 2, + marks=pytest.mark.skipif( + not _provider_installed("airflow.providers.fab"), + reason="fab provider not installed", + ), ), ], ) diff --git a/airflow-core/tests/unit/utils/test_sqlalchemy.py b/airflow-core/tests/unit/utils/test_sqlalchemy.py index 43ead81f46dc6..48c4cb659e4ad 100644 --- a/airflow-core/tests/unit/utils/test_sqlalchemy.py +++ b/airflow-core/tests/unit/utils/test_sqlalchemy.py @@ -23,6 +23,8 @@ from unittest import mock import pytest + +pytest.importorskip("kubernetes", reason="kubernetes package required for sqlalchemy k8s tests") from kubernetes.client import models as k8s from sqlalchemy import text from sqlalchemy.exc import StatementError diff --git a/contributing-docs/07_local_virtualenv.rst b/contributing-docs/07_local_virtualenv.rst index a301b6366cbe6..f40ba43883ae1 100644 --- a/contributing-docs/07_local_virtualenv.rst +++ b/contributing-docs/07_local_virtualenv.rst @@ -181,15 +181,8 @@ will install all dependencies needed to run tests for airflow-core. cd airflow-core uv sync - -TODO(potiuk): This will not work yet - until we move some remaining provider tests from airflow-core. For -now you need to add ``--all-package`` to install all providers and their dependencies. - -.. code:: bash - - cd airflow-core - uv sync --all-packages - +Tests that depend on optional provider packages (e.g. ``kubernetes``, ``celery``) will be +automatically skipped in a core-only environment. Working on individual provider dependencies ........................................... diff --git a/uv.lock b/uv.lock index 254e349f70256..f77daba4703cd 100644 --- a/uv.lock +++ b/uv.lock @@ -1695,12 +1695,6 @@ dev = [ { name = "apache-airflow-core", extra = ["all"] }, { name = "apache-airflow-ctl" }, { name = "apache-airflow-devel-common" }, - { name = "apache-airflow-providers-amazon" }, - { name = "apache-airflow-providers-celery" }, - { name = "apache-airflow-providers-cncf-kubernetes" }, - { name = "apache-airflow-providers-fab" }, - { name = "apache-airflow-providers-ftp" }, - { name = "apache-airflow-providers-git" }, { name = "apache-airflow-task-sdk" }, ] docs = [ @@ -1795,12 +1789,6 @@ dev = [ { name = "apache-airflow-core", extras = ["all"], editable = "airflow-core" }, { name = "apache-airflow-ctl", editable = "airflow-ctl" }, { name = "apache-airflow-devel-common", editable = "devel-common" }, - { name = "apache-airflow-providers-amazon", editable = "providers/amazon" }, - { name = "apache-airflow-providers-celery", editable = "providers/celery" }, - { name = "apache-airflow-providers-cncf-kubernetes", editable = "providers/cncf/kubernetes" }, - { name = "apache-airflow-providers-fab", editable = "providers/fab" }, - { name = "apache-airflow-providers-ftp", editable = "providers/ftp" }, - { name = "apache-airflow-providers-git", editable = "providers/git" }, { name = "apache-airflow-task-sdk", editable = "task-sdk" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }]