AIP-79 Support Airflow 2.x plugins in fast api

.pre-commit-config.yaml

@@ -330,7 +330,7 @@ repos:
           The word(s) should be in lowercase." && exec codespell "$@"' --
         language: python
         types: [text]
-        exclude: ^.*/.*_vendor/|^airflow/www/static/css/material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$|^.*pnpm-lock\.yaml$
+        exclude: ^.*/.*_vendor/|material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$|^.*pnpm-lock\.yaml$
         args:
           - --ignore-words=docs/spelling_wordlist.txt
           - --skip=providers/src/airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md
@@ -622,6 +622,7 @@ repos:
           ^providers/src/airflow/providers/apache/spark/operators/|
           ^providers/src/airflow/providers/exasol/hooks/exasol.py$|
           ^providers/src/airflow/providers/fab/auth_manager/security_manager/|
+          ^providers/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$|
           ^providers/src/airflow/providers/google/cloud/hooks/bigquery.py$|
           ^providers/src/airflow/providers/google/cloud/operators/cloud_build.py$|
           ^providers/src/airflow/providers/google/cloud/operators/dataproc.py$|

airflow/api_fastapi/app.py

@@ -26,6 +26,7 @@
     init_config,
     init_dag_bag,
     init_error_handlers,
+    init_flask_plugins,
     init_plugins,
     init_views,
 )
@@ -67,6 +68,7 @@ def create_app(apps: str = "all") -> FastAPI:
         init_dag_bag(app)
         init_views(app)
         init_plugins(app)
+        init_flask_plugins(app)
         init_error_handlers(app)
         init_auth_manager()
 
@@ -125,6 +127,20 @@ def init_auth_manager() -> BaseAuthManager:
 
 def get_auth_manager() -> BaseAuthManager:
     """Return the auth manager, provided it's been initialized before."""
+    global auth_manager
+    if auth_manager is None:
+        """
+        The auth manager can be init in the main Flask application but also in the mini Flask application
+        in Fab provider.
+        This is temporary, the goal is to remove the main Flask application from core Airflow. Once that done,
+        we'll be able to remove this if because the auth manager will be only init in the min Flask
+        application defined in Fab provider.
+        """
+        from airflow.www.extensions.init_auth_manager import get_auth_manager as get_auth_manager_flask
+
+        if auth_manager_flask := get_auth_manager_flask():
+            auth_manager = auth_manager_flask
+
     if auth_manager is None:
         raise RuntimeError(
             "Auth Manager has not been initialized yet. "

airflow/api_fastapi/core_api/app.py

@@ -18,6 +18,7 @@
 
 import logging
 import os
+import warnings
 from pathlib import Path
 from typing import cast
 
@@ -29,6 +30,7 @@
 from starlette.staticfiles import StaticFiles
 from starlette.templating import Jinja2Templates
 
+from airflow.exceptions import AirflowException
 from airflow.settings import AIRFLOW_PATH
 from airflow.www.extensions.init_dagbag import get_dag_bag
 
@@ -98,6 +100,41 @@ def init_plugins(app: FastAPI) -> None:
         app.mount(url_prefix, subapp)
 
 
+def init_flask_plugins(app: FastAPI) -> None:
+    """Integrate Flask plugins (plugins from Airflow 2)."""
+    from airflow import plugins_manager
+
+    plugins_manager.initialize_flask_plugins()
+
+    # If no Airflow 2.x plugin is in the environment, no need to go further
+    if (
+        not plugins_manager.flask_blueprints
+        and not plugins_manager.flask_appbuilder_views
+        and not plugins_manager.flask_appbuilder_menu_links
+    ):
+        return
+
+    from fastapi.middleware.wsgi import WSGIMiddleware
+
+    try:
+        from airflow.providers.fab.www.app import create_app
+    except ImportError:
+        raise AirflowException(
+            "Some Airflow 2 plugins have been detected in your environment. "
+            "To run them with Airflow 3, you must install the FAB provider in your Airflow environment."
+        )
+
+    warnings.warn(
+        "You have a plugin that is using a FAB view or Flask Blueprint, which was used for the Airflow 2 UI,"
+        "and is now deprecated. Please update your plugin to be compatible with the Airflow 3 UI.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    flask_app = create_app()
+    app.mount("/pluginsv2", WSGIMiddleware(flask_app))
+
+
 def init_config(app: FastAPI) -> None:
     from airflow.configuration import conf
 

airflow/plugins_manager.py

@@ -366,7 +366,7 @@ def ensure_plugins_loaded():
         log.debug("Loading %d plugin(s) took %.2f seconds", len(plugins), timer.duration)
 
 
-def initialize_web_ui_plugins():
+def initialize_flask_plugins():
     """Collect extension points for WEB UI."""
     global plugins
     global flask_blueprints
@@ -577,7 +577,7 @@ def get_plugin_info(attrs_to_dump: Iterable[str] | None = None) -> list[dict[str
     """
     ensure_plugins_loaded()
     integrate_macros_plugins()
-    initialize_web_ui_plugins()
+    initialize_flask_plugins()
     initialize_fastapi_plugins()
     initialize_extra_operators_links_plugins()
     if not attrs_to_dump:

airflow/www/extensions/init_views.py

@@ -132,7 +132,7 @@ def init_plugins(app):
     """Integrate Flask and FAB with plugins."""
     from airflow import plugins_manager
 
-    plugins_manager.initialize_web_ui_plugins()
+    plugins_manager.initialize_flask_plugins()
 
     appbuilder = app.appbuilder
 

airflow/www/views.py

@@ -4296,7 +4296,7 @@ def list(self):
         """List loaded plugins."""
         plugins_manager.ensure_plugins_loaded()
         plugins_manager.initialize_extra_operators_links_plugins()
-        plugins_manager.initialize_web_ui_plugins()
+        plugins_manager.initialize_flask_plugins()
         plugins_manager.initialize_fastapi_plugins()
 
         plugins = []

providers/src/airflow/providers/3rd-party-licenses/LICENSES-ui.txt

@@ -0,0 +1,87 @@
+Apache Airflow
+Copyright 2016-2023 The Apache Software Foundation
+
+This product includes software developed at The Apache Software
+Foundation (http://www.apache.org/).
+
+=======================================================================
+css-loader|5.2.7:
+-----
+MIT
+Copyright JS Foundation and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+webpack-contrib/css-loader
+
+
+moment|2.30.1:
+-----
+MIT
+Copyright (c) JS Foundation and other contributors
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+https://github.com/moment/moment.git
+
+
+moment-timezone|0.5.46:
+-----
+MIT
+The MIT License (MIT)
+
+Copyright (c) JS Foundation and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+https://github.com/moment/moment-timezone.git

providers/src/airflow/providers/fab/auth_manager/api/auth/backend/basic_auth.py

@@ -25,8 +25,8 @@
 from flask_appbuilder.const import AUTH_LDAP
 from flask_login import login_user
 
+from airflow.api_fastapi.app import get_auth_manager
 from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
-from airflow.www.extensions.init_auth_manager import get_auth_manager
 
 if TYPE_CHECKING:
     from airflow.providers.fab.auth_manager.models import User

providers/src/airflow/providers/fab/auth_manager/api/auth/backend/kerberos_auth.py

@@ -26,10 +26,10 @@
 from flask import Response, current_app, g, make_response, request
 from requests_kerberos import HTTPKerberosAuth
 
+from airflow.api_fastapi.app import get_auth_manager
 from airflow.configuration import conf
 from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
 from airflow.utils.net import getfqdn
-from airflow.www.extensions.init_auth_manager import get_auth_manager
 
 if TYPE_CHECKING:
     from airflow.auth.managers.models.base_user import BaseUser

providers/src/airflow/providers/fab/auth_manager/api/auth/backend/session.py

@@ -23,7 +23,7 @@
 
 from flask import Response
 
-from airflow.www.extensions.init_auth_manager import get_auth_manager
+from airflow.api_fastapi.app import get_auth_manager
 
 CLIENT_AUTH: tuple[str, str] | Any | None = None
 

providers/src/airflow/providers/fab/auth_manager/api_endpoints/role_and_permission_endpoint.py

@@ -27,6 +27,7 @@
 from airflow.api_connexion.exceptions import AlreadyExists, BadRequest, NotFound
 from airflow.api_connexion.parameters import check_limit, format_parameters
 from airflow.api_connexion.security import requires_access_custom_view
+from airflow.api_fastapi.app import get_auth_manager
 from airflow.providers.fab.auth_manager.models import Action, Role
 from airflow.providers.fab.auth_manager.schemas.role_and_permission_schema import (
     ActionCollection,
@@ -37,7 +38,6 @@
 )
 from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
 from airflow.security import permissions
-from airflow.www.extensions.init_auth_manager import get_auth_manager
 
 if TYPE_CHECKING:
     from airflow.api_connexion.types import APIResponse, UpdateMask

providers/src/airflow/providers/fab/auth_manager/api_endpoints/user_endpoint.py

@@ -28,6 +28,7 @@
 from airflow.api_connexion.exceptions import AlreadyExists, BadRequest, NotFound, Unknown
 from airflow.api_connexion.parameters import check_limit, format_parameters
 from airflow.api_connexion.security import requires_access_custom_view
+from airflow.api_fastapi.app import get_auth_manager
 from airflow.providers.fab.auth_manager.models import User
 from airflow.providers.fab.auth_manager.schemas.user_schema import (
     UserCollection,
@@ -37,7 +38,6 @@
 )
 from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
 from airflow.security import permissions
-from airflow.www.extensions.init_auth_manager import get_auth_manager
 
 if TYPE_CHECKING:
     from airflow.api_connexion.types import APIResponse, UpdateMask

providers/src/airflow/providers/fab/auth_manager/decorators/auth.py

@@ -26,12 +26,12 @@
 
 from airflow.api_connexion.exceptions import PermissionDenied
 from airflow.api_connexion.security import check_authentication
+from airflow.api_fastapi.app import get_auth_manager
 from airflow.configuration import conf
 from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
 from airflow.utils.airflow_flask_app import AirflowApp
 from airflow.utils.net import get_hostname
 from airflow.www.auth import _has_access
-from airflow.www.extensions.init_auth_manager import get_auth_manager
 
 T = TypeVar("T", bound=Callable)
 

providers/src/airflow/providers/fab/auth_manager/security_manager/override.py

@@ -73,6 +73,7 @@
 from werkzeug.security import check_password_hash, generate_password_hash
 
 from airflow import __version__ as airflow_version
+from airflow.api_fastapi.app import get_auth_manager
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.models import DagBag, DagModel
@@ -107,7 +108,6 @@
 )
 from airflow.providers.fab.auth_manager.views.user_stats import CustomUserStatsChartView
 from airflow.security import permissions
-from airflow.www.extensions.init_auth_manager import get_auth_manager
 from airflow.www.security_manager import AirflowSecurityManagerV2
 from airflow.www.session import AirflowDatabaseSessionInterface
 

providers/src/airflow/providers/fab/www/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.