Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
2107311
refactor: Flag conn-fields in hook but absent from provider.yaml
dabla Jul 9, 2026
43010fa
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 10, 2026
5dab8c9
refactor: Reformatted files
dabla Jul 11, 2026
c3a14a0
refactor: Added missing conn-fields in provider.yaml of Hive, Google,…
dabla Jul 12, 2026
ae2bb88
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 12, 2026
3ed2c5d
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 13, 2026
19686cc
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 13, 2026
bb192c5
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 14, 2026
2d8dec2
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 14, 2026
0fba71b
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 15, 2026
3000ad1
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 15, 2026
5670d2a
refactor: Updated uv.lock
Jul 15, 2026
b59ad32
refactor: Changed order of cloud_environment definition in conn-fields
dabla Jul 15, 2026
01256a0
refactor: Added cloud_environment in provider.yaml for AzureBaseHook
dabla Jul 16, 2026
754536e
refactor: Fixed addition of cloud_environment in get_provider_info
dabla Jul 16, 2026
5d3bb50
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 16, 2026
cd2cb0d
Merge branch 'main' into feature/add-static-check-conn-fields-provide…
dabla Jul 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions providers/apache/hive/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,20 @@ connection-types:
- boolean
- 'null'
default: false
ssl:
label: Ssl
schema:
type:
- boolean
- 'null'
default: true
zoo_keeper_namespace:
label: Zoo Keeper Namespace
schema:
type:
- string
- 'null'
default: 'hiveserver2'
- hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveServer2Hook
hook-name: "Hive Server 2 Thrift"
connection-type: hiveserver2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@ def get_provider_info():
"label": "High Availability mode",
"schema": {"type": ["boolean", "null"], "default": False},
},
"ssl": {"label": "Ssl", "schema": {"type": ["boolean", "null"], "default": True}},
"zoo_keeper_namespace": {
"label": "Zoo Keeper Namespace",
"schema": {"type": ["string", "null"], "default": "hiveserver2"},
},
},
},
{
Expand Down
8 changes: 8 additions & 0 deletions providers/google/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1202,6 +1202,10 @@ connection-types:
schema:
type: ["boolean", "null"]
default: false
quota_project_id:
label: "Quota Project ID"
schema:
type: ["string", "null"]
- hook-class-name: airflow.providers.google.cloud.hooks.spanner.SpannerHook
connection-type: gcpspanner
- hook-class-name: airflow.providers.google.cloud.hooks.dataprep.GoogleDataprepHook
Expand Down Expand Up @@ -1281,6 +1285,10 @@ connection-types:
schema:
type: ["boolean", "null"]
default: false
quota_project_id:
label: "Quota Project ID"
schema:
type: ["string", "null"]
use_legacy_sql:
label: "Use Legacy SQL"
schema:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1434,6 +1434,7 @@ def get_provider_info():
"label": "Anonymous credentials (ignores all other settings)",
"schema": {"type": ["boolean", "null"], "default": False},
},
"quota_project_id": {"label": "Quota Project ID", "schema": {"type": ["string", "null"]}},
},
},
{
Expand Down Expand Up @@ -1512,6 +1513,7 @@ def get_provider_info():
"label": "Anonymous credentials (ignores all other settings)",
"schema": {"type": ["boolean", "null"], "default": False},
},
"quota_project_id": {"label": "Quota Project ID", "schema": {"type": ["string", "null"]}},
"use_legacy_sql": {"label": "Use Legacy SQL", "schema": {"type": ["boolean", "null"]}},
"location": {"label": "Location", "schema": {"type": ["string", "null"]}},
"priority": {
Expand Down
5 changes: 4 additions & 1 deletion providers/microsoft/azure/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,6 @@ transfers:
how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/gcs_to_wasb.rst
python-module: airflow.providers.microsoft.azure.transfers.gcs_to_wasb


connection-types:
- hook-class-name: airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook
hook-name: "Azure"
Expand Down Expand Up @@ -417,6 +416,10 @@ connection-types:
label: Workload Identity Tenant ID
schema:
type: ["string", "null"]
cloud_environment:
label: Azure Cloud Environment
schema:
type: ["string", "null"]
- hook-class-name: airflow.providers.microsoft.azure.hooks.ai_agents.AzureAIAgentsHook
hook-name: "Azure AI Foundry Hosted Agents"
connection-type: azure_ai_agents
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,10 @@ def get_provider_info():
"label": "Workload Identity Tenant ID",
"schema": {"type": ["string", "null"]},
},
"cloud_environment": {
"label": "Azure Cloud Environment",
"schema": {"type": ["string", "null"]},
},
},
},
{
Expand Down
6 changes: 6 additions & 0 deletions providers/snowflake/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,12 @@ connection-types:
- string
- 'null'
format: password
workload_identity_provider:
label: Workload Identity Provider
schema:
type:
- string
- 'null'
insecure_mode:
label: Insecure Mode
schema:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,10 @@ def get_provider_info():
"label": "Private key (Text)",
"schema": {"type": ["string", "null"], "format": "password"},
},
"workload_identity_provider": {
"label": "Workload Identity Provider",
"schema": {"type": ["string", "null"]},
},
"insecure_mode": {
"label": "Insecure Mode",
"schema": {"type": ["boolean", "null"]},
Expand Down
44 changes: 26 additions & 18 deletions scripts/ci/prek/check_provider_conn_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,32 +81,40 @@ def build_mismatch_error(
hook_class_name: str,
) -> str | None:
"""
Check that every key declared in ``conn-fields`` exists in
``get_connection_form_widgets()``.

``conn-fields`` is the new React UI's view of a connection type and is
intentionally a *subset* of the Flask form widgets — fields can be omitted
from ``conn-fields`` on purpose. We therefore only flag keys that appear in
``conn-fields`` but are absent from the hook's form (invalid / stale
declarations). The reverse direction (hook fields not listed in
``conn-fields``) is not an error.

Return an error string when stale keys are found, or ``None`` when the
declared keys are all valid.
Check that ``conn-fields`` and ``get_connection_form_widgets()`` are in sync.

Once a provider declares ``conn-fields``, those declarations must exactly
match the hook's form widgets — we flag both directions:

- Keys present in ``conn-fields`` but absent from the hook → stale / invalid
declarations that should be removed.
- Keys present in the hook but absent from ``conn-fields`` → fields that will
be invisible in the new React connection UI, almost certainly unintentional.

Return an error string when any mismatch is found, or ``None`` when the sets
are identical.
"""
only_in_yaml = yaml_keys - hook_keys
only_in_hook = hook_keys - yaml_keys

if not only_in_yaml:
if not only_in_yaml and not only_in_hook:
return None

lines = [
f"Mismatch between `conn-fields` in {yaml_file_path} and "
f"`{hook_class_name}.get_connection_form_widgets()` "
f"for connection-type '{connection_type}':"
]
lines.append(
" Fields in provider.yaml conn-fields but NOT in get_connection_form_widgets(): "
+ ", ".join(sorted(only_in_yaml))
)
lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from conn-fields in provider.yaml.")
if only_in_yaml:
lines.append(
" Fields in provider.yaml conn-fields but NOT in get_connection_form_widgets(): "
+ ", ".join(sorted(only_in_yaml))
)
lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from conn-fields in provider.yaml.")
if only_in_hook:
lines.append(
" Fields in get_connection_form_widgets() but NOT in provider.yaml conn-fields: "
+ ", ".join(sorted(only_in_hook))
)
lines.append("[yellow]How to fix it[/]: Add the missing key(s) to conn-fields in provider.yaml.")
return "\n".join(lines)
13 changes: 5 additions & 8 deletions scripts/in_container/run_provider_yaml_files_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -481,14 +481,11 @@ def check_hook_class_name_entries_in_connection_types(yaml_files: dict[str, dict
@run_check("Checking that conn-fields in provider.yaml match get_connection_form_widgets() of the hook class")
def check_conn_fields_match_form_widgets(yaml_files: dict[str, dict]) -> tuple[int, int]:
"""
For every connection-type entry whose hook declares ``conn-fields``,
verify that every key in ``conn-fields`` also exists in the hook's
``get_connection_form_widgets()``.

``conn-fields`` is optional and is intentionally allowed to be a *subset*
of the hook's form widgets (the new React UI may expose fewer fields than
the legacy Flask form), so extra hook widgets are not flagged — only
``conn-fields`` keys absent from the hook are reported as stale.
For every connection-type entry that both declares ``conn-fields`` and whose
hook overrides ``get_connection_form_widgets()``, verify the two sets are
identical — stale YAML keys (in ``conn-fields`` but not in the hook) and
missing YAML keys (in the hook but absent from ``conn-fields``) are both
reported as errors.
"""
num_checks = 0
num_errors = 0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,6 @@ class TestBuildMismatchError:
[
pytest.param({"a", "b"}, {"a", "b"}, id="matching-keys"),
pytest.param(set(), set(), id="empty-sets"),
# Hook may have more keys than conn-fields — that is intentional (subset allowed).
pytest.param({"a"}, {"a", "extra_hook"}, id="hook-has-extra-keys-no-error"),
],
)
def test_no_mismatch_returns_none(self, yaml_keys, hook_keys):
Expand All @@ -69,18 +67,26 @@ def test_no_mismatch_returns_none(self, yaml_keys, hook_keys):
{"a"},
["extra_yaml", "NOT in get_connection_form_widgets"],
["NOT in provider.yaml conn-fields"],
id="extra-in-yaml",
id="extra-in-yaml-only",
),
pytest.param(
{"a"},
{"a", "extra_hook"},
["extra_hook", "NOT in provider.yaml conn-fields"],
["NOT in get_connection_form_widgets"],
id="extra-in-hook-only",
),
# only_in_hook no longer triggers an error — only only_in_yaml does
pytest.param(
{"a", "only_yaml"},
{"a", "only_hook"},
[
"only_yaml",
"NOT in get_connection_form_widgets",
"only_hook",
"NOT in provider.yaml conn-fields",
],
["only_hook", "NOT in provider.yaml conn-fields"],
id="both-sides-only-yaml-reported",
[],
id="both-sides-both-reported",
),
],
)
Expand All @@ -103,8 +109,6 @@ class TestCheckConnFieldsForEntry:
pytest.param(None, _skip, id="skip-missing-conn-fields-when-hook-has-no-widgets"),
# Hook with widgets but no conn-fields is allowed: new UI intentionally omits custom fields.
pytest.param(None, _get_keys("field_a"), id="no-conn-fields-with-hook-widgets-is-ok"),
# Hook has extra keys not in conn-fields — allowed (conn-fields is a valid subset).
pytest.param(["a"], _get_keys("a", "extra_hook"), id="hook-extra-keys-no-error"),
],
)
def test_no_errors(self, conn_fields, get_keys):
Expand All @@ -114,6 +118,9 @@ def test_no_errors(self, conn_fields, get_keys):
"conn_fields, get_keys, expected_in_error",
[
pytest.param(["a", "extra"], _get_keys("a"), "extra", id="extra-key-in-yaml"),
pytest.param(
["a"], _get_keys("a", "extra_hook"), "extra_hook", id="hook-has-extra-key-not-in-yaml"
),
pytest.param(["a"], _raise, "boom", id="unexpected-exception-message"),
pytest.param(["a"], _raise, HOOK_CLASS, id="unexpected-exception-mentions-hook-class"),
],
Expand Down
Loading