From be88220e6256f3e51d6e59a479b2f816dd40b60c Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:51:15 +0300 Subject: [PATCH] Keep asyncpg installed by default in the Postgres provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Postgres provider 7.0.0 moved asyncpg from a default dependency to an opt-in extra when it made psycopg3 the default async metadata-DB driver. That default only exists on Airflow 3.4.0+, whose async URL derivation can fall back to asyncpg. Airflow cores older than 3.4.0 — all currently released 3.x lines, which this provider still supports (apache-airflow>=2.11.0) — derive the async metadata-DB URL as postgresql+asyncpg:// unconditionally and have no psycopg fallback, so dropping asyncpg leaves their async engine unable to load its driver. Keep asyncpg as a default dependency until the minimum supported Airflow is 3.4.0, so upgrading the provider does not break the async metadata database on the cores it still claims to support. --- providers/postgres/README.rst | 1 + providers/postgres/docs/changelog.rst | 19 ++++++++++++------- providers/postgres/docs/index.rst | 1 + providers/postgres/pyproject.toml | 7 +++++++ uv.lock | 2 ++ 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/providers/postgres/README.rst b/providers/postgres/README.rst index 6ed7987da4917..b499cc4b8cafd 100644 --- a/providers/postgres/README.rst +++ b/providers/postgres/README.rst @@ -60,6 +60,7 @@ PIP package Version required ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` ``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` ``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` +``asyncpg`` ``>=0.30.0`` ========================================== ====================================== Optional cross provider package dependencies diff --git a/providers/postgres/docs/changelog.rst b/providers/postgres/docs/changelog.rst index 5d2db6e2ccd62..7198d5756d45c 100644 --- a/providers/postgres/docs/changelog.rst +++ b/providers/postgres/docs/changelog.rst @@ -34,14 +34,19 @@ Breaking changes ~~~~~~~~~~~~~~~~ .. note:: - The default async metadata-database driver is now ``psycopg`` (psycopg3), - which is installed by default in place of ``asyncpg``. As a result, the - ``[psycopg]`` optional extra was removed and ``[asyncpg]`` was added. - - Starting with Airflow 3.4.0, the default derived async connection URL changes from + On Airflow 3.4.0 and later, the default async metadata-database driver becomes + ``psycopg`` (psycopg3): the derived async connection URL changes from ``postgresql+asyncpg://`` to ``postgresql+psycopg_async://``, which is safe behind - transaction-mode PgBouncer with no extra configuration. To keep using asyncpg, install - ``apache-airflow-providers-postgres[asyncpg]`` and set + transaction-mode PgBouncer with no extra configuration. ``psycopg[binary]`` is now + installed by default to serve it. + + ``asyncpg`` remains installed by default as well. This provider still supports Airflow + cores older than 3.4.0, which derive the async URL as ``postgresql+asyncpg://`` + unconditionally and have no psycopg fallback — so asyncpg must stay present for them to + work. It will become opt-in-only (via the existing ``[asyncpg]`` extra) in a future + release once the minimum supported Airflow is 3.4.0. + + To keep using asyncpg on Airflow 3.4.0+, set ``[database] sql_alchemy_conn_async = postgresql+asyncpg://...`` explicitly. * ``Switch the default async Postgres driver from asyncpg to psycopg3 (#69089)`` diff --git a/providers/postgres/docs/index.rst b/providers/postgres/docs/index.rst index 3ef0f7798a807..71ed92201ca0c 100644 --- a/providers/postgres/docs/index.rst +++ b/providers/postgres/docs/index.rst @@ -108,6 +108,7 @@ PIP package Version required ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` ``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` ``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` +``asyncpg`` ``>=0.30.0`` ========================================== ====================================== Optional cross provider package dependencies diff --git a/providers/postgres/pyproject.toml b/providers/postgres/pyproject.toml index e946c8201e236..71a9a295b85c4 100644 --- a/providers/postgres/pyproject.toml +++ b/providers/postgres/pyproject.toml @@ -68,6 +68,13 @@ dependencies = [ "psycopg2-binary>=2.9.10; python_version >= '3.13'", "psycopg[binary]>=3.2.9; python_version < '3.14'", "psycopg[binary]>=3.3.3; python_version >= '3.14'", + # asyncpg must stay a default dependency while this provider still supports Airflow + # cores older than 3.4.0: those cores derive the async metadata-DB URL as + # ``postgresql+asyncpg://`` unconditionally and have no psycopg_async fallback, so + # dropping asyncpg breaks their async engine. Make it opt-in-only (via the ``[asyncpg]`` + # extra) only once the minimum supported Airflow is >= 3.4.0; tracked at + # https://github.com/apache/airflow/issues/68453 + "asyncpg>=0.30.0", ] # The optional dependencies should be modified in place in the generated file diff --git a/uv.lock b/uv.lock index 43b6db586d290..9f6d24c6aa667 100644 --- a/uv.lock +++ b/uv.lock @@ -7172,6 +7172,7 @@ dependencies = [ { name = "apache-airflow" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql" }, + { name = "asyncpg" }, { name = "psycopg", extra = ["binary"] }, { name = "psycopg2-binary" }, ] @@ -7223,6 +7224,7 @@ requires-dist = [ { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-microsoft-azure", marker = "extra == 'microsoft-azure'", editable = "providers/microsoft/azure" }, { name = "apache-airflow-providers-openlineage", marker = "extra == 'openlineage'", editable = "providers/openlineage" }, + { name = "asyncpg", specifier = ">=0.30.0" }, { name = "asyncpg", marker = "extra == 'asyncpg'", specifier = ">=0.30.0" }, { name = "pandas", marker = "python_full_version == '3.13.*' and extra == 'pandas'", specifier = ">=2.2.3" }, { name = "pandas", marker = "python_full_version < '3.13' and extra == 'pandas'", specifier = ">=2.1.2" },