Skip to content

Remove unnecessary entries in get_provider_info and update the schema #48849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Apr 8, 2025
Merged

Remove unnecessary entries in get_provider_info and update the schema #48849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 198 additions & 9 deletions airflow-core/src/airflow/provider_info.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"type": "string"
},
"deprecated": {
"description": "The hook-class-names property has been deprecated in favour of connection-types which is more performant version allowing to only import individual Hooks rather than all hooks at once",
"description": "The hook-class-names property has been deprecated in favor of connection-types which is more performant version allowing to only import individual Hooks rather than all hooks at once",
"deprecatedVersion": "2.2.0"
}
},
Expand All @@ -32,7 +32,182 @@
"type": "string"
}
},
"integrations": {
"description": "Array of integrations provided by the provider",
"type": "array",
"items": {
"type": "object",
"properties": {
"integration-name": {
"description": "Name of the integrations to expose by the provider",
"type": "string"
},
"external-doc-url": {
"description": "URL that describes the integration",
"type": "string"
},
"logo": {
"description": "URL or path on the airflow side where you can find the logo",
Comment thread
potiuk marked this conversation as resolved.
"type": "string"
},
"tags": {
"description": "Tags describing the integration (free-form)",
"type": "array",
"items": {
"type": "string"
}
}
}

}
},
"operators": {
"description": "List of operators provided by the integration",
"type": "array",
"items": {
"type": "object",
"properties": {
"integration-name": {
"description": "Name of the integration",
"type": "string"
},
"python-modules": {
"description": "List of modules where operators are found",
"type" : "array",
"items": {
"type": "string"
}
}
}
}
},
"sensors": {
"description": "List of sensors provided by the integration",
"type": "array",
"items": {
"type": "object",
"properties": {
"integration-name": {
"description": "Name of the integration",
"type": "string"
},
"python-modules": {
"description": "List of modules where sensors are found",
"type" : "array",
"items": {
"type": "string"
}
}
}
}
},
"hooks": {
"description": "List of hooks provided by the integration",
"type": "array",
"items": {
"type": "object",
"properties": {
"integration-name": {
"description": "Name of the integration",
"type": "string"
},
"python-modules": {
"description": "List of modules where hooks are found",
"type" : "array",
"items": {
"type": "string"
}
}
}
}
},
"asset-uris": {
"description": "List of asset uris provided by the provider",
"type": "array",
"items": {
"type": "object",
"properties": {
"schemes": {
"description": "Array of schemes supported",
"type": "array",
"items" : {
"type": "string"
}
},
"handler": {
"description": "Handler used to handle the asset",
"anyOf": [
{ "type": "string" },
{ "type": "null" }
]
},
"factory": {
"description": "Factory to create the asset",
"type": "string"
},
"to_openlineage_converter": {
"description": "Converter to open-lineage event",
"type": "string"
}
},
"required": ["schemes"]
}
},
"dataset-uris": {
"description": "List of asset uris provided by the provider",
"type": "array",
"items": {
"type": "object",
"properties": {
"schemes": {
"description": "Array of schemes supported",
"type": "array",
"items" : {
"type": "string"
}
},
"handler": {
"description": "Handler used to handle the asset",
"anyOf": [
{ "type": "string" },
{ "type": "null" }
]
},
"factory": {
"description": "Factory to create the asset",
"type": "string"
},
"to_openlineage_converter": {
"description": "Converter to open-lineage event",
"type": "string"
}
},
"required": ["schemes"]
},
"deprecated": {
"description": "The dataset-uris property has been deprecated in favor of asset-uris in airflow 3.0",
"deprecatedVersion": "3.0.0"
}
},
"dialects": {
"description": "List of dialects the provider provides",
"type": "array",
"items": {
"type": "object",
"properties": {
"dialect-type": {
"description": "Type of SQL dialect",
"type": "string"
},
"dialect-class-name": {
"description": "Class name that implements the dialect",
"type": "string"
}
}
}
},
"transfers": {
"description": "List of transfer operators the provider provides",
"type": "array",
"items": {
"type": "object",
Expand All @@ -43,15 +218,15 @@
},
"source-integration-name": {
"type": "string",
"description": "Integration name. It must have a matching item in the 'integration' section of any provider."
"description": "Integration name. It must have a matching item in the 'integration' section of any provider"
},
"target-integration-name": {
"type": "string",
"description": "Target integration name. It must have a matching item in the 'integration' section of any provider."
"description": "Target integration name. It must have a matching item in the 'integration' section of any provider"
},
"python-module": {
"type": "string",
"description": "List of python modules containing the transfers."
"description": "List of python modules containing the transfers"
}
},
"additionalProperties": false,
Expand All @@ -69,10 +244,10 @@
"properties": {
"integration-name": {
"type": "string",
"description": "Integration name. It must have a matching item in the 'integration' section of any provider."
"description": "Integration name. It must have a matching item in the 'integration' section of any provider"
},
"python-modules": {
"description": "List of Python modules containing the triggers.",
"description": "List of Python modules containing the triggers",
"type": "array",
"items": {
"type": "string"
Expand All @@ -93,10 +268,10 @@
"properties": {
"integration-name": {
"type": "string",
"description": "Integration name. It must have a matching item in the 'integration' section of any provider."
"description": "Integration name. It must have a matching item in the 'integration' section of any provider"
},
"python-modules": {
"description": "List of python modules containing the bundles.",
"description": "List of python modules containing the bundles",
"type": "array",
"items": {
"type": "string"
Expand All @@ -112,7 +287,7 @@
},
"connection-types": {
"type": "array",
"description": "Map of connection types mapped to hook class names.",
"description": "Map of connection types mapped to hook class names",
"items": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -227,6 +402,20 @@
"type": "string"
}
}
},
"plugins": {
"type": "array",
"description": "Plugins provided by the provider",
"items": {
"name": {
"type": "string",
"description": "Name of the plugin"
},
"plugin-class": {
"type": "string",
"description": "Class to instantiate the plugin"
}
}
}
},
"definitions": {
Expand Down
4 changes: 2 additions & 2 deletions dev/breeze/src/airflow_breeze/commands/release_management_commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -962,7 +962,7 @@ def prepare_provider_documentation(
f"Update release notes for package '{provider_id}' ",
skip_printing_title=only_min_version_update,
):
if not only_min_version_update:
if not only_min_version_update and not reapply_templates_only:
get_console().print("Updating documentation for the latest release version.")
with_breaking_changes, maybe_with_new_features = update_release_notes(
provider_id,
Expand All @@ -977,7 +977,7 @@ def prepare_provider_documentation(
with_breaking_changes=with_breaking_changes,
maybe_with_new_features=maybe_with_new_features,
)
if not only_min_version_update:
if not only_min_version_update and not reapply_templates_only:
with ci_group(
f"Updates changelog for last release of package '{provider_id}'",
skip_printing_title=only_min_version_update,
Expand Down
12 changes: 12 additions & 0 deletions dev/breeze/src/airflow_breeze/utils/packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,17 @@ def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) ->
raise SystemExit(1)


def filter_provider_info_data(provider_info: dict[str, Any]) -> dict[str, Any]:
json_schema_dict = json.loads(PROVIDER_RUNTIME_DATA_SCHEMA_PATH.read_text())
runtime_properties = json_schema_dict["properties"].keys()
return_dict = {
property: provider_info[property]
for property in provider_info.keys()
if property in runtime_properties
}
return return_dict


def get_provider_info_dict(provider_id: str) -> dict[str, Any]:
"""Retrieves provider info from the provider yaml file.

Expand All @@ -236,6 +247,7 @@ def get_provider_info_dict(provider_id: str) -> dict[str, Any]:
"""
provider_yaml_dict = get_provider_distributions_metadata().get(provider_id)
if provider_yaml_dict:
provider_yaml_dict = filter_provider_info_data(provider_yaml_dict)
validate_provider_info_with_runtime_schema(provider_yaml_dict)
return provider_yaml_dict or {}

Expand Down
6 changes: 3 additions & 3 deletions dev/breeze/tests/test_packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,7 @@ def test_get_provider_info_dict():
assert provider_info_dict["name"] == "Amazon"
assert provider_info_dict["package-name"] == "apache-airflow-providers-amazon"
assert "Amazon" in provider_info_dict["description"]
assert provider_info_dict["state"] == "ready"
assert provider_info_dict["filesystems"] == ["airflow.providers.amazon.aws.fs.s3"]
assert len(provider_info_dict["versions"]) > 45
assert len(provider_info_dict["dependencies"]) > 10
assert len(provider_info_dict["integrations"]) > 35
assert len(provider_info_dict["hooks"]) > 30
assert len(provider_info_dict["triggers"]) > 15
Expand All @@ -304,3 +301,6 @@ def test_get_provider_info_dict():
assert len(provider_info_dict["logging"]) > 1
assert len(provider_info_dict["config"].keys()) > 1
assert len(provider_info_dict["executors"]) > 0
assert len(provider_info_dict["dataset-uris"]) > 0
assert len(provider_info_dict["dataset-uris"]) > 0
assert len(provider_info_dict["asset-uris"]) > 0
12 changes: 10 additions & 2 deletions devel-common/src/sphinx_exts/exampleinclude.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,12 +137,14 @@ def register_source(app, env, modname):
:param modname: name of the module to load
:return: True if the code is registered successfully, False otherwise
"""
if modname is None:
return False
entry = env._viewcode_modules.get(modname, None)
if entry is False:
print(f"[{modname}] Entry is false for ")
return False

code_tags = app.emit_firstresult("viewcode-find-source", modname)

if code_tags is None:
try:
analyzer = ModuleAnalyzer.for_module(modname)
Expand Down Expand Up @@ -237,7 +239,13 @@ def doctree_read(app, doctree):
relative_path = os.path.relpath(
filepath, os.path.commonprefix([app.config.exampleinclude_sourceroot, filepath])
)
modname = relative_path.replace("/", ".")[:-3]
if relative_path.endswith(".py"):
modname = relative_path.replace("/", ".")[-3]
split_modname = modname.split(".")
if "src" in split_modname:
modname = ".".join(split_modname[split_modname.index("src") + 1 :])
else:
modname = None
show_button = register_source(app, env, modname)
onlynode = create_node(env, relative_path, show_button)

Expand Down
18 changes: 12 additions & 6 deletions devel-common/src/tests_common/test_utils/providers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,17 @@ def get_provider_version(provider_name):
return semver.VersionInfo.parse(info.version)


def get_provider_min_airflow_version(provider_name):
from airflow.providers_manager import ProvidersManager
def get_provider_min_airflow_version(provider_name: str) -> tuple[int, ...]:
from importlib import metadata

from packaging.version import Version

p = ProvidersManager()
deps = p.providers[provider_name].data["dependencies"]
deps = metadata.requires(provider_name)
if not deps:
raise RuntimeError(f"The provider should have dependencies: {provider_name}")
airflow_dep = next(x for x in deps if x.startswith("apache-airflow"))
min_airflow_version = tuple(map(int, airflow_dep.split(">=")[1].split(".")))
return min_airflow_version
if not airflow_dep:
raise RuntimeError(
f"The provider should have `apache-airflow>=` in their dependencies: {provider_name}"
)
return Version(airflow_dep.split(">=")[1]).release
Loading