diff --git a/docs/build_docs.py b/docs/build_docs.py
index 2799a46169d78..6f4acc184ba3b 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -115,11 +115,6 @@ def clean_files() -> None:
     os.makedirs(_API_DIR, exist_ok=True)
     os.makedirs(_BUILD_DIR, exist_ok=True)
     print(f"Recreated content of the ${_BUILD_DIR} and ${_API_DIR} folders")
-    # Bugs in sphinx-autoapi using metaclasses prevent us from upgrading to 1.3
-    # which has implicit namespace support. Until that time, we make it look
-    # like a real package for building docs
-    with open(PROVIDER_INIT_FILE, "wt"):
-        pass
 
 
 def display_errors_summary() -> None:
@@ -608,6 +603,7 @@ def check_spelling() -> None:
     :return:
     """
     extensions_to_use = [
+        'provider_init_hack',
         "sphinxarg.ext",
         "autoapi.extension",
         "sphinxcontrib.spelling",
@@ -727,21 +723,18 @@ def print_build_errors_and_exit(message) -> None:
 Invitation link: https://s.apache.org/airflow-slack\
 """
 
-try:
-    print_build_errors_and_exit("The documentation has errors. Fix them to build documentation.")
-
-    if not args.docs_only:
-        check_spelling()
-        print_build_errors_and_exit("The documentation has spelling errors. Fix them to build documentation.")
-
-    if not args.spellcheck_only:
-        build_sphinx_docs()
-        check_guide_links_in_operator_descriptions()
-        check_class_links_in_operators_and_hooks_ref()
-        check_guide_links_in_operators_and_hooks_ref()
-        check_enforce_code_block()
-        check_exampleinclude_for_example_dags()
-        check_google_guides()
-        print_build_errors_and_exit("The documentation has errors.")
-finally:
-    shutil.rmtree(PROVIDER_INIT_FILE, ignore_errors=True)
+print_build_errors_and_exit("The documentation has errors. Fix them to build documentation.")
+
+if not args.docs_only:
+    check_spelling()
+    print_build_errors_and_exit("The documentation has spelling errors. Fix them to build documentation.")
+
+if not args.spellcheck_only:
+    build_sphinx_docs()
+    check_guide_links_in_operator_descriptions()
+    check_class_links_in_operators_and_hooks_ref()
+    check_guide_links_in_operators_and_hooks_ref()
+    check_enforce_code_block()
+    check_exampleinclude_for_example_dags()
+    check_google_guides()
+    print_build_errors_and_exit("The documentation has errors.")
diff --git a/docs/conf.py b/docs/conf.py
index 19cdc051d56a6..e071158a8688c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -78,6 +78,7 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'provider_init_hack',
     'sphinx.ext.autodoc',
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
@@ -424,6 +425,7 @@ def _get_rst_filepath_from_path(filepath: str):
     '*/airflow/kubernetes/kubernetes_request_factory/*',
     '*/_internal*',
     '*/node_modules/*',
+    '*/example_dags/*,',
     '*/migrations/*',
 ]
 # Keep the AutoAPI generated files on the filesystem after the run.
diff --git a/docs/exts/provider_init_hack.py b/docs/exts/provider_init_hack.py
new file mode 100644
index 0000000000000..0d885591453b7
--- /dev/null
+++ b/docs/exts/provider_init_hack.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Bugs in sphinx-autoapi using metaclasses prevent us from upgrading to 1.3
+which has implicit namespace support. Until that time, we make it look
+like a real package for building docs
+"""
+import os
+
+from sphinx.application import Sphinx
+
+ROOT_PROJECT_DIR = os.path.abspath(
+    os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir)
+)
+
+PROVIDER_INIT_FILE = os.path.join(ROOT_PROJECT_DIR, "airflow", "providers", "__init__.py")
+
+
+def _create_init_py(app, config):
+    del app
+    del config
+    with open(PROVIDER_INIT_FILE, "wt"):
+        pass
+
+
+def _delete_init_py(app, exception):
+    del app
+    del exception
+    if os.path.exists(PROVIDER_INIT_FILE):
+        os.remove(PROVIDER_INIT_FILE)
+
+
+def setup(app: Sphinx):
+    """
+    Sets the plugin up and returns configuration of the plugin.
+
+    :param app: application.
+    :return json description of the configuration that is needed by the plugin.
+    """
+    app.connect("config-inited", _create_init_py)
+    app.connect("build-finished", _delete_init_py)
+
+    return {"version": "builtin", "parallel_read_safe": True, "parallel_write_safe": True}