diff --git a/providers/postgres/README.rst b/providers/postgres/README.rst index 3f8b66862114f..4e6f61a59f5ea 100644 --- a/providers/postgres/README.rst +++ b/providers/postgres/README.rst @@ -58,7 +58,8 @@ PIP package Version required ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"`` ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` -``asyncpg`` ``>=0.30.0`` +``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` +``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` ========================================== ====================================== Optional cross provider package dependencies @@ -89,11 +90,11 @@ Optional dependencies Extra Dependencies =================== ============================================================================================================================================================ ``amazon`` ``apache-airflow-providers-amazon>=2.6.0`` +``asyncpg`` ``asyncpg>=0.30.0`` ``microsoft.azure`` ``apache-airflow-providers-microsoft-azure>=12.8.0`` ``openlineage`` ``apache-airflow-providers-openlineage`` ``pandas`` ``pandas>=2.1.2; python_version <"3.13"``, ``pandas>=2.2.3; python_version >="3.13" and python_version <"3.14"``, ``pandas>=2.3.3; python_version >="3.14"`` ``polars`` ``polars>=1.26.0`` -``psycopg`` ``psycopg[binary]>=3.2.9; python_version < '3.14'``, ``psycopg[binary]>=3.3.3; python_version >= '3.14'`` ``sqlalchemy`` ``sqlalchemy>=1.4.54`` =================== ============================================================================================================================================================ diff --git a/providers/postgres/docs/changelog.rst b/providers/postgres/docs/changelog.rst index 934ded27a23a7..eff74f3268483 100644 --- a/providers/postgres/docs/changelog.rst +++ b/providers/postgres/docs/changelog.rst @@ -27,6 +27,21 @@ Changelog --------- + +Breaking changes +~~~~~~~~~~~~~~~~ + +.. note:: + The default async metadata-database driver is now ``psycopg`` (psycopg3), + which is installed by default in place of ``asyncpg``. As a result, the + ``[psycopg]`` optional extra was removed and ``[asyncpg]`` was added. + + Starting with Airflow 3.4.0, the default derived async connection URL changes from + ``postgresql+asyncpg://`` to ``postgresql+psycopg_async://``, which is safe behind + transaction-mode PgBouncer with no extra configuration. To keep using asyncpg, install + ``apache-airflow-providers-postgres[asyncpg]`` and set + ``[database] sql_alchemy_conn_async = postgresql+asyncpg://...`` explicitly. + 6.8.0 ..... diff --git a/providers/postgres/docs/index.rst b/providers/postgres/docs/index.rst index eef9e8e5eeee3..b10aa40df7646 100644 --- a/providers/postgres/docs/index.rst +++ b/providers/postgres/docs/index.rst @@ -106,7 +106,8 @@ PIP package Version required ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"`` ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` -``asyncpg`` ``>=0.30.0`` +``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` +``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` ========================================== ====================================== Optional cross provider package dependencies diff --git a/providers/postgres/pyproject.toml b/providers/postgres/pyproject.toml index 83b2b92f61053..8adda0b57802b 100644 --- a/providers/postgres/pyproject.toml +++ b/providers/postgres/pyproject.toml @@ -62,9 +62,12 @@ dependencies = [ "apache-airflow>=2.11.0", "apache-airflow-providers-common-compat>=1.12.0", "apache-airflow-providers-common-sql>=1.32.0", + # psycopg2 remains the sync driver for now; its removal and the sync migration to + # psycopg3 are tracked at https://github.com/apache/airflow/issues/68453 "psycopg2-binary>=2.9.9; python_version < '3.13'", "psycopg2-binary>=2.9.10; python_version >= '3.13'", - "asyncpg>=0.30.0", + "psycopg[binary]>=3.2.9; python_version < '3.14'", + "psycopg[binary]>=3.3.3; python_version >= '3.14'", ] # The optional dependencies should be modified in place in the generated file @@ -73,6 +76,9 @@ dependencies = [ "amazon" = [ "apache-airflow-providers-amazon>=2.6.0", ] +"asyncpg" = [ + "asyncpg>=0.30.0", +] "microsoft.azure" = [ "apache-airflow-providers-microsoft-azure>=12.8.0" ] @@ -87,10 +93,6 @@ dependencies = [ "polars" = [ "polars>=1.26.0" ] -"psycopg" = [ - "psycopg[binary]>=3.2.9; python_version < '3.14'", - "psycopg[binary]>=3.3.3; python_version >= '3.14'", -] "sqlalchemy" = [ "sqlalchemy>=1.4.54" ] @@ -108,6 +110,7 @@ dev = [ # Additional devel dependencies (do not remove this line and add extra development dependencies) "apache-airflow-providers-common-sql[pandas]", "apache-airflow-providers-common-sql[polars]", + "apache-airflow-providers-postgres[asyncpg]", "apache-airflow-providers-postgres[sqlalchemy]" ] diff --git a/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py b/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py index 77ca0537b934d..be6519ddade24 100644 --- a/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py +++ b/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py @@ -70,30 +70,14 @@ class CompatConnection(Protocol): - """Protocol for type hinting psycopg2 and psycopg3 connection objects.""" + """Protocol for the common interface shared by psycopg2 and psycopg3 connection objects.""" def cursor(self, *args, **kwargs) -> Any: ... def commit(self) -> None: ... def close(self) -> None: ... - - # Context manager support - def __enter__(self) -> CompatConnection: ... + def __enter__(self) -> Any: ... def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... - # Common properties - @property - def notices(self) -> list[Any]: ... - - # psycopg3 specific (optional) - @property - def adapters(self) -> Any: ... - - @property - def row_factory(self) -> Any: ... - - # Optional method for psycopg3 - def add_notice_handler(self, handler: Any) -> None: ... - class PostgresHook(DbApiHook): """ diff --git a/uv.lock b/uv.lock index bf13cb8afb9b1..11bbace1114d2 100644 --- a/uv.lock +++ b/uv.lock @@ -7150,7 +7150,7 @@ dependencies = [ { name = "apache-airflow" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql" }, - { name = "asyncpg" }, + { name = "psycopg", extra = ["binary"] }, { name = "psycopg2-binary" }, ] @@ -7158,6 +7158,9 @@ dependencies = [ amazon = [ { name = "apache-airflow-providers-amazon" }, ] +asyncpg = [ + { name = "asyncpg" }, +] microsoft-azure = [ { name = "apache-airflow-providers-microsoft-azure" }, ] @@ -7170,9 +7173,6 @@ pandas = [ polars = [ { name = "polars" }, ] -psycopg = [ - { name = "psycopg", extra = ["binary"] }, -] sqlalchemy = [ { name = "sqlalchemy" }, ] @@ -7186,7 +7186,7 @@ dev = [ { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-microsoft-azure" }, { name = "apache-airflow-providers-openlineage" }, - { name = "apache-airflow-providers-postgres", extra = ["sqlalchemy"] }, + { name = "apache-airflow-providers-postgres", extra = ["asyncpg", "sqlalchemy"] }, { name = "apache-airflow-task-sdk" }, ] docs = [ @@ -7201,18 +7201,18 @@ requires-dist = [ { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-microsoft-azure", marker = "extra == 'microsoft-azure'", editable = "providers/microsoft/azure" }, { name = "apache-airflow-providers-openlineage", marker = "extra == 'openlineage'", editable = "providers/openlineage" }, - { name = "asyncpg", specifier = ">=0.30.0" }, + { name = "asyncpg", marker = "extra == 'asyncpg'", specifier = ">=0.30.0" }, { name = "pandas", marker = "python_full_version == '3.13.*' and extra == 'pandas'", specifier = ">=2.2.3" }, { name = "pandas", marker = "python_full_version < '3.13' and extra == 'pandas'", specifier = ">=2.1.2" }, { name = "pandas", marker = "python_full_version >= '3.14' and extra == 'pandas'", specifier = ">=2.3.3" }, { name = "polars", marker = "extra == 'polars'", specifier = ">=1.26.0" }, - { name = "psycopg", extras = ["binary"], marker = "python_full_version >= '3.14' and extra == 'psycopg'", specifier = ">=3.3.3" }, - { name = "psycopg", extras = ["binary"], marker = "python_full_version < '3.14' and extra == 'psycopg'", specifier = ">=3.2.9" }, + { name = "psycopg", extras = ["binary"], marker = "python_full_version < '3.14'", specifier = ">=3.2.9" }, + { name = "psycopg", extras = ["binary"], marker = "python_full_version >= '3.14'", specifier = ">=3.3.3" }, { name = "psycopg2-binary", marker = "python_full_version < '3.13'", specifier = ">=2.9.9" }, { name = "psycopg2-binary", marker = "python_full_version >= '3.13'", specifier = ">=2.9.10" }, { name = "sqlalchemy", marker = "extra == 'sqlalchemy'", specifier = ">=1.4.54" }, ] -provides-extras = ["amazon", "microsoft-azure", "openlineage", "pandas", "polars", "psycopg", "sqlalchemy"] +provides-extras = ["amazon", "asyncpg", "microsoft-azure", "openlineage", "pandas", "polars", "sqlalchemy"] [package.metadata.requires-dev] dev = [ @@ -7225,6 +7225,7 @@ dev = [ { name = "apache-airflow-providers-common-sql", extras = ["polars"], editable = "providers/common/sql" }, { name = "apache-airflow-providers-microsoft-azure", editable = "providers/microsoft/azure" }, { name = "apache-airflow-providers-openlineage", editable = "providers/openlineage" }, + { name = "apache-airflow-providers-postgres", extras = ["asyncpg"], editable = "providers/postgres" }, { name = "apache-airflow-providers-postgres", extras = ["sqlalchemy"], editable = "providers/postgres" }, { name = "apache-airflow-task-sdk", editable = "task-sdk" }, ]