From 98e1ae9e771767770fe2cf62d9f2b3b374105fa9 Mon Sep 17 00:00:00 2001 From: PoAn Yang Date: Mon, 13 Jul 2026 20:56:01 +0900 Subject: [PATCH 1/2] Fix Azure Data Factory hook broken by azure-mgmt-datafactory 10 Signed-off-by: PoAn Yang --- .../microsoft/azure/hooks/data_factory.py | 63 +++++- .../azure/hooks/test_data_factory.py | 196 +++++++++++++++--- 2 files changed, 228 insertions(+), 31 deletions(-) diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py index dd408bed51555..ec41347f12e88 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py @@ -39,13 +39,15 @@ from functools import wraps from typing import IO, TYPE_CHECKING, Any, TypeVar, cast +from azure.core import MatchConditions from azure.identity import ClientSecretCredential, DefaultAzureCredential from azure.identity.aio import ( ClientSecretCredential as AsyncClientSecretCredential, DefaultAzureCredential as AsyncDefaultAzureCredential, ) -from azure.mgmt.datafactory import DataFactoryManagementClient +from azure.mgmt.datafactory import DataFactoryManagementClient, __version__ as _ADF_SDK_VERSION from azure.mgmt.datafactory.aio import DataFactoryManagementClient as AsyncDataFactoryManagementClient +from packaging.version import Version from airflow.providers.common.compat.connection import get_async_connection from airflow.providers.common.compat.sdk import AirflowException, BaseHook @@ -71,6 +73,30 @@ Credentials = ClientSecretCredential | DefaultAzureCredential AsyncCredentials = AsyncClientSecretCredential | AsyncDefaultAzureCredential +# azure-mgmt-datafactory 10 replaced the if_match/if_none_match operation parameters +# with azure-core's keyword-only etag + match_condition pair (IfNotModified sends +# If-Match, IfModified sends If-None-Match). Older SDKs silently ignore the new-style +# arguments and drop the conditional header, so the calling convention has to be +# picked by the installed major version. +# https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/datafactory/azure-mgmt-datafactory/CHANGELOG.md#breaking-changes +_ADF_SDK_V10_PLUS = Version(_ADF_SDK_VERSION).major >= 10 + + +def _build_if_match_kwargs(if_match: str | None) -> dict[str, Any]: + if _ADF_SDK_V10_PLUS: + return {"etag": if_match, "match_condition": MatchConditions.IfNotModified if if_match else None} + return {"if_match": if_match} + + +def _build_if_none_match_kwargs(if_none_match: str | None) -> dict[str, Any]: + if _ADF_SDK_V10_PLUS: + return { + "etag": if_none_match, + "match_condition": MatchConditions.IfModified if if_none_match else None, + } + return {"if_none_match": if_none_match} + + T = TypeVar("T", bound=Any) @@ -278,7 +304,7 @@ def update_factory( raise AirflowException(f"Factory {factory!r} does not exist.") return self.get_conn().factories.create_or_update( - resource_group_name, factory_name, factory, if_match, **config + resource_group_name, factory_name, factory, **_build_if_match_kwargs(if_match), **config ) @provide_targeted_factory @@ -339,7 +365,11 @@ def get_linked_service( :return: The linked service. """ return self.get_conn().linked_services.get( - resource_group_name, factory_name, linked_service_name, if_none_match, **config + resource_group_name, + factory_name, + linked_service_name, + **_build_if_none_match_kwargs(if_none_match), + **config, ) def _linked_service_exists(self, resource_group_name, factory_name, linked_service_name) -> bool: @@ -549,7 +579,11 @@ def get_dataflow( :return: The DataFlowResource. """ return self.get_conn().data_flows.get( - resource_group_name, factory_name, dataflow_name, if_none_match, **config + resource_group_name, + factory_name, + dataflow_name, + **_build_if_none_match_kwargs(if_none_match), + **config, ) def _dataflow_exists( @@ -597,7 +631,12 @@ def update_dataflow( raise AirflowException(f"Dataflow {dataflow_name!r} does not exist.") return self.get_conn().data_flows.create_or_update( - resource_group_name, factory_name, dataflow_name, dataflow, if_match, **config + resource_group_name, + factory_name, + dataflow_name, + dataflow, + **_build_if_match_kwargs(if_match), + **config, ) @provide_targeted_factory @@ -627,7 +666,12 @@ def create_dataflow( raise AirflowException(f"Dataflow {dataflow_name!r} already exists.") return self.get_conn().data_flows.create_or_update( - resource_group_name, factory_name, dataflow_name, dataflow, if_match, **config + resource_group_name, + factory_name, + dataflow_name, + dataflow, + **_build_if_match_kwargs(if_match), + **config, ) @provide_targeted_factory @@ -923,7 +967,12 @@ def update_trigger( raise AirflowException(f"Trigger {trigger_name!r} does not exist.") return self.get_conn().triggers.create_or_update( - resource_group_name, factory_name, trigger_name, trigger, if_match, **config + resource_group_name, + factory_name, + trigger_name, + trigger, + **_build_if_match_kwargs(if_match), + **config, ) @provide_targeted_factory diff --git a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py index 92608f6101a94..3ff1b408115cc 100644 --- a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py +++ b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py @@ -17,11 +17,23 @@ from __future__ import annotations import os +from collections import namedtuple from unittest import mock from unittest.mock import MagicMock, PropertyMock, patch import pytest +from azure.core import MatchConditions +from azure.core.pipeline.transport import HttpTransport +from azure.mgmt.datafactory import DataFactoryManagementClient as SyncDataFactoryManagementClient from azure.mgmt.datafactory.aio import DataFactoryManagementClient +from azure.mgmt.datafactory.models import ( + DataFlowResource, + Factory, + MappingDataFlow, + ScheduleTrigger, + ScheduleTriggerRecurrence, + TriggerResource, +) from airflow.models.connection import Connection from airflow.providers.common.compat.sdk import AirflowException @@ -30,6 +42,8 @@ AzureDataFactoryHook, AzureDataFactoryPipelineRunException, AzureDataFactoryPipelineRunStatus, + _build_if_match_kwargs, + _build_if_none_match_kwargs, get_field, provide_targeted_factory, ) @@ -149,6 +163,78 @@ def hook(): return client +class _RequestCaptured(Exception): + """Raised by _CaptureTransport so the request is inspected instead of sent.""" + + +class _CaptureTransport(HttpTransport): + """Record the request the real installed SDK builds, without any network I/O.""" + + def __init__(self): + self.last_request = None + + def send(self, request, **kwargs): + self.last_request = request + raise _RequestCaptured + + def open(self): + pass + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + pass + + +class _FakeCredential: + def get_token(self, *scopes, **kwargs): + return namedtuple("AccessToken", ["token", "expires_on"])("fake-token", 2**33) + + +def _make_wire_client(transport: _CaptureTransport) -> SyncDataFactoryManagementClient: + return SyncDataFactoryManagementClient(_FakeCredential(), "subscription-id", transport=transport) + + +def _get_conditional_headers(transport: _CaptureTransport) -> dict[str, str]: + return { + key.lower(): value + for key, value in transport.last_request.headers.items() + if key.lower() in ("if-match", "if-none-match") + } + + +@pytest.mark.parametrize( + ("sdk_v10_plus", "if_match", "expected_kwargs"), + [ + (True, '"abc123"', {"etag": '"abc123"', "match_condition": MatchConditions.IfNotModified}), + (True, None, {"etag": None, "match_condition": None}), + (False, '"abc123"', {"if_match": '"abc123"'}), + (False, None, {"if_match": None}), + ], +) +def test_build_if_match_kwargs(sdk_v10_plus, if_match, expected_kwargs): + with mock.patch(f"{MODULE}._ADF_SDK_V10_PLUS", sdk_v10_plus): + assert _build_if_match_kwargs(if_match) == expected_kwargs + + +@pytest.mark.parametrize( + ("sdk_v10_plus", "if_none_match", "expected_kwargs"), + [ + (True, '"abc123"', {"etag": '"abc123"', "match_condition": MatchConditions.IfModified}), + (True, None, {"etag": None, "match_condition": None}), + (False, '"abc123"', {"if_none_match": '"abc123"'}), + (False, None, {"if_none_match": None}), + ], +) +def test_build_if_none_match_kwargs(sdk_v10_plus, if_none_match, expected_kwargs): + with mock.patch(f"{MODULE}._ADF_SDK_V10_PLUS", sdk_v10_plus): + assert _build_if_none_match_kwargs(if_none_match) == expected_kwargs + + def parametrize(explicit_factory, implicit_factory): def wrapper(func): return pytest.mark.parametrize( @@ -242,12 +328,20 @@ def test_create_factory(hook: AzureDataFactoryHook): hook._conn.factories.create_or_update.assert_called_with(RESOURCE_GROUP, FACTORY, MODEL) -def test_update_factory(hook: AzureDataFactoryHook): - with patch.object(hook, "_factory_exists") as mock_factory_exists: - mock_factory_exists.return_value = True - hook.update_factory(MODEL, RESOURCE_GROUP, FACTORY) +@pytest.mark.parametrize( + ("if_match", "expected_headers"), + [('"abc123"', {"if-match": '"abc123"'}), (None, {})], +) +def test_update_factory(hook: AzureDataFactoryHook, if_match, expected_headers): + transport = _CaptureTransport() + with ( + patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), + patch.object(hook, "_factory_exists", return_value=True), + pytest.raises(_RequestCaptured), + ): + hook.update_factory(Factory(location="eastus"), RESOURCE_GROUP, FACTORY, if_match) - hook._conn.factories.create_or_update.assert_called_with(RESOURCE_GROUP, FACTORY, MODEL, None) + assert _get_conditional_headers(transport) == expected_headers def test_update_factory_non_existent(hook: AzureDataFactoryHook): @@ -264,10 +358,19 @@ def test_delete_factory(hook: AzureDataFactoryHook): hook._conn.factories.delete.assert_called_with(RESOURCE_GROUP, FACTORY) -def test_get_linked_service(hook: AzureDataFactoryHook): - hook.get_linked_service(NAME, RESOURCE_GROUP, FACTORY) +@pytest.mark.parametrize( + ("if_none_match", "expected_headers"), + [('"abc123"', {"if-none-match": '"abc123"'}), (None, {})], +) +def test_get_linked_service(hook: AzureDataFactoryHook, if_none_match, expected_headers): + transport = _CaptureTransport() + with ( + patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), + pytest.raises(_RequestCaptured), + ): + hook.get_linked_service(NAME, RESOURCE_GROUP, FACTORY, if_none_match) - hook._conn.linked_services.get.assert_called_with(RESOURCE_GROUP, FACTORY, NAME, None) + assert _get_conditional_headers(transport) == expected_headers def test_create_linked_service(hook: AzureDataFactoryHook): @@ -332,24 +435,55 @@ def test_delete_dataset(hook: AzureDataFactoryHook): hook._conn.datasets.delete.assert_called_with(RESOURCE_GROUP, FACTORY, NAME) -def test_get_dataflow(hook: AzureDataFactoryHook): - hook.get_dataflow(NAME, RESOURCE_GROUP, FACTORY) +@pytest.mark.parametrize( + ("if_none_match", "expected_headers"), + [('"abc123"', {"if-none-match": '"abc123"'}), (None, {})], +) +def test_get_dataflow(hook: AzureDataFactoryHook, if_none_match, expected_headers): + transport = _CaptureTransport() + with ( + patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), + pytest.raises(_RequestCaptured), + ): + hook.get_dataflow(NAME, RESOURCE_GROUP, FACTORY, if_none_match) - hook._conn.data_flows.get.assert_called_with(RESOURCE_GROUP, FACTORY, NAME, None) + assert _get_conditional_headers(transport) == expected_headers -def test_create_dataflow(hook: AzureDataFactoryHook): - hook.create_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY) +@pytest.mark.parametrize( + ("if_match", "expected_headers"), + [('"abc123"', {"if-match": '"abc123"'}), (None, {})], +) +def test_create_dataflow(hook: AzureDataFactoryHook, if_match, expected_headers): + transport = _CaptureTransport() + with ( + patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), + patch.object(hook, "_dataflow_exists", return_value=False), + pytest.raises(_RequestCaptured), + ): + hook.create_dataflow( + NAME, DataFlowResource(properties=MappingDataFlow()), RESOURCE_GROUP, FACTORY, if_match + ) - hook._conn.data_flows.create_or_update.assert_called_with(RESOURCE_GROUP, FACTORY, NAME, MODEL, None) + assert _get_conditional_headers(transport) == expected_headers -def test_update_dataflow(hook: AzureDataFactoryHook): - with patch.object(hook, "_dataflow_exists") as mock_dataflow_exists: - mock_dataflow_exists.return_value = True - hook.update_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY) +@pytest.mark.parametrize( + ("if_match", "expected_headers"), + [('"abc123"', {"if-match": '"abc123"'}), (None, {})], +) +def test_update_dataflow(hook: AzureDataFactoryHook, if_match, expected_headers): + transport = _CaptureTransport() + with ( + patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), + patch.object(hook, "_dataflow_exists", return_value=True), + pytest.raises(_RequestCaptured), + ): + hook.update_dataflow( + NAME, DataFlowResource(properties=MappingDataFlow()), RESOURCE_GROUP, FACTORY, if_match + ) - hook._conn.data_flows.create_or_update.assert_called_with(RESOURCE_GROUP, FACTORY, NAME, MODEL, None) + assert _get_conditional_headers(transport) == expected_headers def test_update_dataflow_non_existent(hook: AzureDataFactoryHook): @@ -473,12 +607,26 @@ def test_create_trigger(hook: AzureDataFactoryHook): hook._conn.triggers.create_or_update.assert_called_with(RESOURCE_GROUP, FACTORY, NAME, MODEL) -def test_update_trigger(hook: AzureDataFactoryHook): - with patch.object(hook, "_trigger_exists") as mock_trigger_exists: - mock_trigger_exists.return_value = True - hook.update_trigger(NAME, MODEL, RESOURCE_GROUP, FACTORY) +@pytest.mark.parametrize( + ("if_match", "expected_headers"), + [('"abc123"', {"if-match": '"abc123"'}), (None, {})], +) +def test_update_trigger(hook: AzureDataFactoryHook, if_match, expected_headers): + transport = _CaptureTransport() + with ( + patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), + patch.object(hook, "_trigger_exists", return_value=True), + pytest.raises(_RequestCaptured), + ): + hook.update_trigger( + NAME, + TriggerResource(properties=ScheduleTrigger(recurrence=ScheduleTriggerRecurrence())), + RESOURCE_GROUP, + FACTORY, + if_match, + ) - hook._conn.triggers.create_or_update.assert_called_with(RESOURCE_GROUP, FACTORY, NAME, MODEL, None) + assert _get_conditional_headers(transport) == expected_headers def test_update_trigger_non_existent(hook: AzureDataFactoryHook): From 0bdf13c86024e9473ffefb4b5e8e5eb85329823b Mon Sep 17 00:00:00 2001 From: PoAn Yang Date: Wed, 15 Jul 2026 14:41:39 +0900 Subject: [PATCH 2/2] Fix Azure Data Factory hook broken by azure-mgmt-datafactory 10 Signed-off-by: PoAn Yang --- providers/microsoft/azure/README.rst | 2 +- providers/microsoft/azure/docs/index.rst | 2 +- providers/microsoft/azure/pyproject.toml | 2 +- .../microsoft/azure/hooks/data_factory.py | 49 ++--- .../azure/hooks/test_data_factory.py | 207 ++++-------------- uv.lock | 8 +- 6 files changed, 72 insertions(+), 198 deletions(-) diff --git a/providers/microsoft/azure/README.rst b/providers/microsoft/azure/README.rst index 63f0e97e18160..be404f636ee4a 100644 --- a/providers/microsoft/azure/README.rst +++ b/providers/microsoft/azure/README.rst @@ -74,7 +74,7 @@ PIP package Version required ``azure-synapse-artifacts`` ``>=0.17.0`` ``azure-storage-file-datalake`` ``>=12.9.1`` ``azure-kusto-data`` ``>=4.1.0,!=5.0.0`` -``azure-mgmt-datafactory`` ``>=2.0.0`` +``azure-mgmt-datafactory`` ``>=10.0.0`` ``azure-mgmt-containerregistry`` ``>=8.0.0`` ``azure-mgmt-compute`` ``>=33.0.0`` ``azure-mgmt-containerinstance`` ``>=10.1.0`` diff --git a/providers/microsoft/azure/docs/index.rst b/providers/microsoft/azure/docs/index.rst index 0ad9ca9512f32..b326188e24e90 100644 --- a/providers/microsoft/azure/docs/index.rst +++ b/providers/microsoft/azure/docs/index.rst @@ -128,7 +128,7 @@ PIP package Version required ``azure-synapse-artifacts`` ``>=0.17.0`` ``azure-storage-file-datalake`` ``>=12.9.1`` ``azure-kusto-data`` ``>=4.1.0,!=5.0.0`` -``azure-mgmt-datafactory`` ``>=2.0.0`` +``azure-mgmt-datafactory`` ``>=10.0.0`` ``azure-mgmt-containerregistry`` ``>=8.0.0`` ``azure-mgmt-compute`` ``>=33.0.0`` ``azure-mgmt-containerinstance`` ``>=10.1.0`` diff --git a/providers/microsoft/azure/pyproject.toml b/providers/microsoft/azure/pyproject.toml index 9b2d117593b5a..91327c2c74d20 100644 --- a/providers/microsoft/azure/pyproject.toml +++ b/providers/microsoft/azure/pyproject.toml @@ -84,7 +84,7 @@ dependencies = [ "azure-storage-file-datalake>=12.9.1", # azure-kusto-data 5.0.0 pins requests to a specific version which makes resolving dependencies harder "azure-kusto-data>=4.1.0,!=5.0.0", - "azure-mgmt-datafactory>=2.0.0", + "azure-mgmt-datafactory>=10.0.0", "azure-mgmt-containerregistry>=8.0.0", "azure-mgmt-compute>=33.0.0", "azure-mgmt-containerinstance>=10.1.0", diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py index ec41347f12e88..61ae3c8848b9c 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py @@ -45,9 +45,8 @@ ClientSecretCredential as AsyncClientSecretCredential, DefaultAzureCredential as AsyncDefaultAzureCredential, ) -from azure.mgmt.datafactory import DataFactoryManagementClient, __version__ as _ADF_SDK_VERSION +from azure.mgmt.datafactory import DataFactoryManagementClient from azure.mgmt.datafactory.aio import DataFactoryManagementClient as AsyncDataFactoryManagementClient -from packaging.version import Version from airflow.providers.common.compat.connection import get_async_connection from airflow.providers.common.compat.sdk import AirflowException, BaseHook @@ -73,30 +72,6 @@ Credentials = ClientSecretCredential | DefaultAzureCredential AsyncCredentials = AsyncClientSecretCredential | AsyncDefaultAzureCredential -# azure-mgmt-datafactory 10 replaced the if_match/if_none_match operation parameters -# with azure-core's keyword-only etag + match_condition pair (IfNotModified sends -# If-Match, IfModified sends If-None-Match). Older SDKs silently ignore the new-style -# arguments and drop the conditional header, so the calling convention has to be -# picked by the installed major version. -# https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/datafactory/azure-mgmt-datafactory/CHANGELOG.md#breaking-changes -_ADF_SDK_V10_PLUS = Version(_ADF_SDK_VERSION).major >= 10 - - -def _build_if_match_kwargs(if_match: str | None) -> dict[str, Any]: - if _ADF_SDK_V10_PLUS: - return {"etag": if_match, "match_condition": MatchConditions.IfNotModified if if_match else None} - return {"if_match": if_match} - - -def _build_if_none_match_kwargs(if_none_match: str | None) -> dict[str, Any]: - if _ADF_SDK_V10_PLUS: - return { - "etag": if_none_match, - "match_condition": MatchConditions.IfModified if if_none_match else None, - } - return {"if_none_match": if_none_match} - - T = TypeVar("T", bound=Any) @@ -304,7 +279,12 @@ def update_factory( raise AirflowException(f"Factory {factory!r} does not exist.") return self.get_conn().factories.create_or_update( - resource_group_name, factory_name, factory, **_build_if_match_kwargs(if_match), **config + resource_group_name, + factory_name, + factory, + etag=if_match, + match_condition=MatchConditions.IfNotModified if if_match else None, + **config, ) @provide_targeted_factory @@ -368,7 +348,8 @@ def get_linked_service( resource_group_name, factory_name, linked_service_name, - **_build_if_none_match_kwargs(if_none_match), + etag=if_none_match, + match_condition=MatchConditions.IfModified if if_none_match else None, **config, ) @@ -582,7 +563,8 @@ def get_dataflow( resource_group_name, factory_name, dataflow_name, - **_build_if_none_match_kwargs(if_none_match), + etag=if_none_match, + match_condition=MatchConditions.IfModified if if_none_match else None, **config, ) @@ -635,7 +617,8 @@ def update_dataflow( factory_name, dataflow_name, dataflow, - **_build_if_match_kwargs(if_match), + etag=if_match, + match_condition=MatchConditions.IfNotModified if if_match else None, **config, ) @@ -670,7 +653,8 @@ def create_dataflow( factory_name, dataflow_name, dataflow, - **_build_if_match_kwargs(if_match), + etag=if_match, + match_condition=MatchConditions.IfNotModified if if_match else None, **config, ) @@ -971,7 +955,8 @@ def update_trigger( factory_name, trigger_name, trigger, - **_build_if_match_kwargs(if_match), + etag=if_match, + match_condition=MatchConditions.IfNotModified if if_match else None, **config, ) diff --git a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py index 3ff1b408115cc..1b64d29c3ce64 100644 --- a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py +++ b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py @@ -17,23 +17,12 @@ from __future__ import annotations import os -from collections import namedtuple from unittest import mock from unittest.mock import MagicMock, PropertyMock, patch import pytest from azure.core import MatchConditions -from azure.core.pipeline.transport import HttpTransport -from azure.mgmt.datafactory import DataFactoryManagementClient as SyncDataFactoryManagementClient from azure.mgmt.datafactory.aio import DataFactoryManagementClient -from azure.mgmt.datafactory.models import ( - DataFlowResource, - Factory, - MappingDataFlow, - ScheduleTrigger, - ScheduleTriggerRecurrence, - TriggerResource, -) from airflow.models.connection import Connection from airflow.providers.common.compat.sdk import AirflowException @@ -42,8 +31,6 @@ AzureDataFactoryHook, AzureDataFactoryPipelineRunException, AzureDataFactoryPipelineRunStatus, - _build_if_match_kwargs, - _build_if_none_match_kwargs, get_field, provide_targeted_factory, ) @@ -163,78 +150,6 @@ def hook(): return client -class _RequestCaptured(Exception): - """Raised by _CaptureTransport so the request is inspected instead of sent.""" - - -class _CaptureTransport(HttpTransport): - """Record the request the real installed SDK builds, without any network I/O.""" - - def __init__(self): - self.last_request = None - - def send(self, request, **kwargs): - self.last_request = request - raise _RequestCaptured - - def open(self): - pass - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, *exc): - pass - - -class _FakeCredential: - def get_token(self, *scopes, **kwargs): - return namedtuple("AccessToken", ["token", "expires_on"])("fake-token", 2**33) - - -def _make_wire_client(transport: _CaptureTransport) -> SyncDataFactoryManagementClient: - return SyncDataFactoryManagementClient(_FakeCredential(), "subscription-id", transport=transport) - - -def _get_conditional_headers(transport: _CaptureTransport) -> dict[str, str]: - return { - key.lower(): value - for key, value in transport.last_request.headers.items() - if key.lower() in ("if-match", "if-none-match") - } - - -@pytest.mark.parametrize( - ("sdk_v10_plus", "if_match", "expected_kwargs"), - [ - (True, '"abc123"', {"etag": '"abc123"', "match_condition": MatchConditions.IfNotModified}), - (True, None, {"etag": None, "match_condition": None}), - (False, '"abc123"', {"if_match": '"abc123"'}), - (False, None, {"if_match": None}), - ], -) -def test_build_if_match_kwargs(sdk_v10_plus, if_match, expected_kwargs): - with mock.patch(f"{MODULE}._ADF_SDK_V10_PLUS", sdk_v10_plus): - assert _build_if_match_kwargs(if_match) == expected_kwargs - - -@pytest.mark.parametrize( - ("sdk_v10_plus", "if_none_match", "expected_kwargs"), - [ - (True, '"abc123"', {"etag": '"abc123"', "match_condition": MatchConditions.IfModified}), - (True, None, {"etag": None, "match_condition": None}), - (False, '"abc123"', {"if_none_match": '"abc123"'}), - (False, None, {"if_none_match": None}), - ], -) -def test_build_if_none_match_kwargs(sdk_v10_plus, if_none_match, expected_kwargs): - with mock.patch(f"{MODULE}._ADF_SDK_V10_PLUS", sdk_v10_plus): - assert _build_if_none_match_kwargs(if_none_match) == expected_kwargs - - def parametrize(explicit_factory, implicit_factory): def wrapper(func): return pytest.mark.parametrize( @@ -329,19 +244,17 @@ def test_create_factory(hook: AzureDataFactoryHook): @pytest.mark.parametrize( - ("if_match", "expected_headers"), - [('"abc123"', {"if-match": '"abc123"'}), (None, {})], + ("if_match", "expected_match_condition"), + [(None, None), ("etag-value", MatchConditions.IfNotModified)], ) -def test_update_factory(hook: AzureDataFactoryHook, if_match, expected_headers): - transport = _CaptureTransport() - with ( - patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), - patch.object(hook, "_factory_exists", return_value=True), - pytest.raises(_RequestCaptured), - ): - hook.update_factory(Factory(location="eastus"), RESOURCE_GROUP, FACTORY, if_match) +def test_update_factory(hook: AzureDataFactoryHook, if_match, expected_match_condition): + with patch.object(hook, "_factory_exists") as mock_factory_exists: + mock_factory_exists.return_value = True + hook.update_factory(MODEL, RESOURCE_GROUP, FACTORY, if_match) - assert _get_conditional_headers(transport) == expected_headers + hook._conn.factories.create_or_update.assert_called_with( + RESOURCE_GROUP, FACTORY, MODEL, etag=if_match, match_condition=expected_match_condition + ) def test_update_factory_non_existent(hook: AzureDataFactoryHook): @@ -359,18 +272,15 @@ def test_delete_factory(hook: AzureDataFactoryHook): @pytest.mark.parametrize( - ("if_none_match", "expected_headers"), - [('"abc123"', {"if-none-match": '"abc123"'}), (None, {})], + ("if_none_match", "expected_match_condition"), + [(None, None), ("etag-value", MatchConditions.IfModified)], ) -def test_get_linked_service(hook: AzureDataFactoryHook, if_none_match, expected_headers): - transport = _CaptureTransport() - with ( - patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), - pytest.raises(_RequestCaptured), - ): - hook.get_linked_service(NAME, RESOURCE_GROUP, FACTORY, if_none_match) +def test_get_linked_service(hook: AzureDataFactoryHook, if_none_match, expected_match_condition): + hook.get_linked_service(NAME, RESOURCE_GROUP, FACTORY, if_none_match) - assert _get_conditional_headers(transport) == expected_headers + hook._conn.linked_services.get.assert_called_with( + RESOURCE_GROUP, FACTORY, NAME, etag=if_none_match, match_condition=expected_match_condition + ) def test_create_linked_service(hook: AzureDataFactoryHook): @@ -436,54 +346,41 @@ def test_delete_dataset(hook: AzureDataFactoryHook): @pytest.mark.parametrize( - ("if_none_match", "expected_headers"), - [('"abc123"', {"if-none-match": '"abc123"'}), (None, {})], + ("if_none_match", "expected_match_condition"), + [(None, None), ("etag-value", MatchConditions.IfModified)], ) -def test_get_dataflow(hook: AzureDataFactoryHook, if_none_match, expected_headers): - transport = _CaptureTransport() - with ( - patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), - pytest.raises(_RequestCaptured), - ): - hook.get_dataflow(NAME, RESOURCE_GROUP, FACTORY, if_none_match) +def test_get_dataflow(hook: AzureDataFactoryHook, if_none_match, expected_match_condition): + hook.get_dataflow(NAME, RESOURCE_GROUP, FACTORY, if_none_match) - assert _get_conditional_headers(transport) == expected_headers + hook._conn.data_flows.get.assert_called_with( + RESOURCE_GROUP, FACTORY, NAME, etag=if_none_match, match_condition=expected_match_condition + ) @pytest.mark.parametrize( - ("if_match", "expected_headers"), - [('"abc123"', {"if-match": '"abc123"'}), (None, {})], + ("if_match", "expected_match_condition"), + [(None, None), ("etag-value", MatchConditions.IfNotModified)], ) -def test_create_dataflow(hook: AzureDataFactoryHook, if_match, expected_headers): - transport = _CaptureTransport() - with ( - patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), - patch.object(hook, "_dataflow_exists", return_value=False), - pytest.raises(_RequestCaptured), - ): - hook.create_dataflow( - NAME, DataFlowResource(properties=MappingDataFlow()), RESOURCE_GROUP, FACTORY, if_match - ) +def test_create_dataflow(hook: AzureDataFactoryHook, if_match, expected_match_condition): + hook.create_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY, if_match) - assert _get_conditional_headers(transport) == expected_headers + hook._conn.data_flows.create_or_update.assert_called_with( + RESOURCE_GROUP, FACTORY, NAME, MODEL, etag=if_match, match_condition=expected_match_condition + ) @pytest.mark.parametrize( - ("if_match", "expected_headers"), - [('"abc123"', {"if-match": '"abc123"'}), (None, {})], + ("if_match", "expected_match_condition"), + [(None, None), ("etag-value", MatchConditions.IfNotModified)], ) -def test_update_dataflow(hook: AzureDataFactoryHook, if_match, expected_headers): - transport = _CaptureTransport() - with ( - patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), - patch.object(hook, "_dataflow_exists", return_value=True), - pytest.raises(_RequestCaptured), - ): - hook.update_dataflow( - NAME, DataFlowResource(properties=MappingDataFlow()), RESOURCE_GROUP, FACTORY, if_match - ) +def test_update_dataflow(hook: AzureDataFactoryHook, if_match, expected_match_condition): + with patch.object(hook, "_dataflow_exists") as mock_dataflow_exists: + mock_dataflow_exists.return_value = True + hook.update_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY, if_match) - assert _get_conditional_headers(transport) == expected_headers + hook._conn.data_flows.create_or_update.assert_called_with( + RESOURCE_GROUP, FACTORY, NAME, MODEL, etag=if_match, match_condition=expected_match_condition + ) def test_update_dataflow_non_existent(hook: AzureDataFactoryHook): @@ -608,25 +505,17 @@ def test_create_trigger(hook: AzureDataFactoryHook): @pytest.mark.parametrize( - ("if_match", "expected_headers"), - [('"abc123"', {"if-match": '"abc123"'}), (None, {})], + ("if_match", "expected_match_condition"), + [(None, None), ("etag-value", MatchConditions.IfNotModified)], ) -def test_update_trigger(hook: AzureDataFactoryHook, if_match, expected_headers): - transport = _CaptureTransport() - with ( - patch.object(hook, "get_conn", return_value=_make_wire_client(transport)), - patch.object(hook, "_trigger_exists", return_value=True), - pytest.raises(_RequestCaptured), - ): - hook.update_trigger( - NAME, - TriggerResource(properties=ScheduleTrigger(recurrence=ScheduleTriggerRecurrence())), - RESOURCE_GROUP, - FACTORY, - if_match, - ) +def test_update_trigger(hook: AzureDataFactoryHook, if_match, expected_match_condition): + with patch.object(hook, "_trigger_exists") as mock_trigger_exists: + mock_trigger_exists.return_value = True + hook.update_trigger(NAME, MODEL, RESOURCE_GROUP, FACTORY, if_match) - assert _get_conditional_headers(transport) == expected_headers + hook._conn.triggers.create_or_update.assert_called_with( + RESOURCE_GROUP, FACTORY, NAME, MODEL, etag=if_match, match_condition=expected_match_condition + ) def test_update_trigger_non_existent(hook: AzureDataFactoryHook): diff --git a/uv.lock b/uv.lock index aeb4d3f9de6eb..b5e8f242a2771 100644 --- a/uv.lock +++ b/uv.lock @@ -6364,7 +6364,7 @@ requires-dist = [ { name = "azure-mgmt-containerinstance", specifier = ">=10.1.0" }, { name = "azure-mgmt-containerregistry", specifier = ">=8.0.0" }, { name = "azure-mgmt-cosmosdb", specifier = ">=3.0.0" }, - { name = "azure-mgmt-datafactory", specifier = ">=2.0.0" }, + { name = "azure-mgmt-datafactory", specifier = ">=10.0.0" }, { name = "azure-mgmt-datalake-store", specifier = ">=0.5.0" }, { name = "azure-mgmt-resource", specifier = ">=2.2.0" }, { name = "azure-mgmt-storage", specifier = ">=16.0.0" }, @@ -9726,16 +9726,16 @@ wheels = [ [[package]] name = "azure-mgmt-datafactory" -version = "9.3.0" +version = "10.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "azure-mgmt-core" }, { name = "isodate" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e8/5e/c28a8a50885185a9a06ff659594fec24f1a840622f21883b376358dd1188/azure_mgmt_datafactory-9.3.0.tar.gz", hash = "sha256:f5fdd5cd416f0ed71dfedf05dc7677b8f0e52f3428fd5b17b04c9200dd8d36b3", size = 494270, upload-time = "2026-03-11T07:52:35.206Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/f9/81397c6afd0fed926ffe3093d108d0392745bddc05c5dfc69880dfb70bb2/azure_mgmt_datafactory-10.0.0.tar.gz", hash = "sha256:c83eeb2b092c1f1e5bb2499ffbba6fd7809a1274739930f9db1e336273405587", size = 717334, upload-time = "2026-07-08T09:40:00.573Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/35/ea6a784652119a7373a730d0eb87d5c0b7d153721d7e23c8e9b76547aca4/azure_mgmt_datafactory-9.3.0-py3-none-any.whl", hash = "sha256:fddb855a27e3f7b78328f184df146d71e433e1dfb9cc4923ea503c53813f0504", size = 570055, upload-time = "2026-03-11T07:52:37.337Z" }, + { url = "https://files.pythonhosted.org/packages/bb/41/b5a74df49ed17aa27361fc55fd88a07bfe7c7a55bcb5ed210c0e0e712d57/azure_mgmt_datafactory-10.0.0-py3-none-any.whl", hash = "sha256:d0e93194c9e1cd820d2ed4f92813902af89cf4421c02e72a729ae32d87171198", size = 601269, upload-time = "2026-07-08T09:40:02.559Z" }, ] [[package]]