diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 56273d8db9aa6..53ff431a8ec61 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -97,7 +97,8 @@ jobs: # yamllint disable rule:line-length skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder && '--skip-write-to-stable-folder' || '' }} default-python-version: "3.10" - registry-providers: ${{ steps.parameters.outputs.registry-providers }} + registry-providers: ${{ steps.derive_registry_inputs.outputs.registry-providers }} + registry-full-build: ${{ steps.derive_registry_inputs.outputs.registry-full-build }} if: contains(fromJSON('[ "ashb", "bugraoz93", @@ -114,6 +115,19 @@ jobs: "vatsrahul1001", ]'), github.event.sender.login) steps: + - name: "Checkout for wave provider derivation" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + fetch-tags: true + fetch-depth: 0 + - name: "Derive registry trigger inputs" + id: derive_registry_inputs + env: + INCLUDE_DOCS: ${{ inputs.include-docs }} + REF: ${{ inputs.ref }} + run: python3 dev/registry/derive_wave_providers.py - name: "Input parameters summary" shell: bash id: parameters @@ -165,27 +179,6 @@ jobs: echo "airflow-version=no-airflow" >> ${GITHUB_OUTPUT} echo "airflow-base-version=no-airflow" >> ${GITHUB_OUTPUT} fi - # Extract provider IDs for registry update - # include-docs contains short names like "amazon", "common.ai", "openlineage" - # or full package names like "apache-airflow-providers-amazon" - REGISTRY_PROVIDERS="" - for pkg in ${INCLUDE_DOCS}; do - case "${pkg}" in - apache-airflow-providers-*) - PROVIDER_ID="${pkg#apache-airflow-providers-}" - REGISTRY_PROVIDERS="${REGISTRY_PROVIDERS:+${REGISTRY_PROVIDERS} }${PROVIDER_ID}" - ;; - apache-airflow|all-providers|apache-airflow-providers|helm-chart|docker-stack|apache-airflow-ctl|apache-airflow-task-sdk) - # Not a provider package, skip - ;; - *) - # Short name like "amazon" or "common.ai" -- convert dots to hyphens - PROVIDER_ID="${pkg//./-}" - REGISTRY_PROVIDERS="${REGISTRY_PROVIDERS:+${REGISTRY_PROVIDERS} }${PROVIDER_ID}" - ;; - esac - done - echo "registry-providers=${REGISTRY_PROVIDERS}" >> ${GITHUB_OUTPUT} build-docs: needs: [build-info] @@ -434,7 +427,7 @@ jobs: update-registry: needs: [publish-docs-to-s3, build-info] - if: needs.build-info.outputs.registry-providers != '' + if: needs.build-info.outputs.registry-providers != '' || needs.build-info.outputs.registry-full-build == 'true' name: "Update Provider Registry" permissions: contents: read diff --git a/dev/registry/derive_wave_providers.py b/dev/registry/derive_wave_providers.py new file mode 100644 index 0000000000000..7a473e8a9ebe8 --- /dev/null +++ b/dev/registry/derive_wave_providers.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Derive registry trigger inputs for publish-docs-to-s3.yml. + +Implements the gating logic for astronomer/issues-airflow#1305. + +Wave dispatches send INCLUDE_DOCS="all-providers apache-airflow-providers" +(see dev/breeze/src/airflow_breeze/commands/workflow_commands.py:188-195 +and dev/README_RELEASE_PROVIDERS.md). When a meta-token is present and the +ref is a wave tag (providers/YYYY-MM-DD), this script derives the wave's +actual provider list from per-provider tags reachable from the wave ref +but not from the previous wave tag, so registry-build runs incrementally +on just those providers. Falls back to a full rebuild for non-wave refs +or when the derivation produces an empty list. + +Tag-push ordering: README_RELEASE_PROVIDERS.md flows the wave-tag push +BEFORE provider RC tags. Today publish-docs-to-s3 is dispatched manually +AFTER all provider tags exist, so the derivation always has its inputs. +Any future auto-trigger on wave-tag push must address this race. + +Inputs (env): INCLUDE_DOCS, REF, GITHUB_OUTPUT. +Outputs (written to GITHUB_OUTPUT): + registry-providers space-separated provider IDs, or "" for full build + registry-full-build "true" for full rebuild, "false" otherwise +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys + +WAVE_TAG_RE = re.compile(r"^providers/[0-9]{4}-[0-9]{2}-[0-9]{2}$") +WAVE_TAG_GLOB = "providers/[0-9]*-*-*" +PROVIDER_TAG_GLOB = "providers-*/*" +PROVIDER_TAG_RE = re.compile(r"^providers-(?P.+?)/(?P.+)$") +RC_SUFFIX_RE = re.compile(r"rc[0-9]+$") + +META_TOKENS = frozenset({"all", "all-providers", "apache-airflow-providers"}) +NON_PROVIDER_TOKENS = frozenset( + { + "apache-airflow", + "helm-chart", + "docker-stack", + "apache-airflow-ctl", + "apache-airflow-task-sdk", + } +) +PROVIDER_PREFIX = "apache-airflow-providers-" + + +def _git(*args: str) -> list[str]: + result = subprocess.run(["git", *args], capture_output=True, text=True, check=True) + return [line for line in result.stdout.splitlines() if line] + + +def previous_wave_tag(ref: str, wave_tags: list[str]) -> str | None: + """Return the wave tag immediately preceding ref, or None. + + Filters input through WAVE_TAG_RE because the git glob `providers/[0-9]*-*-*` + is broader than the strict `providers/YYYY-MM-DD` shape and matches + unrelated tags (e.g. `providers/2.11/2026-02-16` for an Airflow 2.11 + release with a date suffix). + """ + valid = [t for t in wave_tags if WAVE_TAG_RE.match(t)] + try: + idx = valid.index(ref) + except ValueError: + return None + return valid[idx + 1] if idx + 1 < len(valid) else None + + +def wave_provider_ids(tags: list[str]) -> list[str]: + """Provider IDs from per-provider tags, dropping rc tags. Sorted.""" + ids: set[str] = set() + for tag in tags: + if RC_SUFFIX_RE.search(tag): + continue + match = PROVIDER_TAG_RE.match(tag) + if match: + ids.add(match.group("id")) + return sorted(ids) + + +def parse_explicit_packages(include_docs: str) -> list[str]: + """Per-package incremental path -- INCLUDE_DOCS without meta-tokens.""" + providers: list[str] = [] + seen: set[str] = set() + for token in include_docs.split(): + if token in NON_PROVIDER_TOKENS or token in META_TOKENS: + continue + if token.startswith(PROVIDER_PREFIX): + provider_id = token[len(PROVIDER_PREFIX) :] + else: + provider_id = token.replace(".", "-") + if provider_id and provider_id not in seen: + seen.add(provider_id) + providers.append(provider_id) + return providers + + +def derive(include_docs: str, ref: str, *, git_runner=_git) -> tuple[str, bool, str]: + """Return (registry_providers, full_build, log_message). + + Pure function -- pass git_runner=fake for tests. + """ + tokens = include_docs.split() + has_meta = any(t in META_TOKENS for t in tokens) + + if not has_meta: + providers = parse_explicit_packages(include_docs) + return " ".join(providers), False, "" + + if not WAVE_TAG_RE.match(ref): + return "", True, f"REF={ref} is not a wave-tag pattern; using full rebuild for meta-token dispatch." + + wave_tags = git_runner("tag", "--list", WAVE_TAG_GLOB, "--sort=-creatordate") + prev = previous_wave_tag(ref, wave_tags) + if not prev: + return ( + "", + True, + f"::warning::No predecessor wave tag found before {ref}; falling back to full rebuild.", + ) + + raw_tags = git_runner("tag", "--merged", ref, "--no-merged", prev, "--list", PROVIDER_TAG_GLOB) + providers = wave_provider_ids(raw_tags) + if not providers: + return ( + "", + True, + f"::warning::Wave ref {ref} has no new per-provider tags since {prev}; falling back to full rebuild.", + ) + + joined = " ".join(providers) + return joined, False, f"Derived wave providers ({prev} -> {ref}): {joined}" + + +def main() -> int: + include_docs = os.environ.get("INCLUDE_DOCS", "") + ref = os.environ.get("REF", "") + output_path = os.environ.get("GITHUB_OUTPUT") + if not output_path: + print("GITHUB_OUTPUT not set", file=sys.stderr) + return 1 + + registry_providers, full_build, log = derive(include_docs, ref) + if log: + print(log) + with open(output_path, "a", encoding="utf-8") as fh: + fh.write(f"registry-providers={registry_providers}\n") + fh.write(f"registry-full-build={'true' if full_build else 'false'}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev/registry/tests/test_derive_wave_providers.py b/dev/registry/tests/test_derive_wave_providers.py new file mode 100644 index 0000000000000..32a35d93b9520 --- /dev/null +++ b/dev/registry/tests/test_derive_wave_providers.py @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for dev/registry/derive_wave_providers.py.""" + +from __future__ import annotations + +import pytest +from derive_wave_providers import derive + + +def fake_git(wave_tags=None, provider_tags=None): + """Build a git_runner that returns canned outputs based on the args.""" + wave_tags = wave_tags or [] + provider_tags = provider_tags or [] + + def runner(*args): + if args[:2] == ("tag", "--list") and "providers/[0-9]*-*-*" in args: + return list(wave_tags) + if args[:2] == ("tag", "--merged"): + return list(provider_tags) + raise AssertionError(f"unexpected git invocation: {args}") + + return runner + + +WAVE_TAGS = [ + "providers/2026-04-21", + "providers/2026-04-12", + "providers/2026-03-17", +] + +WAVE_PROVIDER_TAGS = [ + "providers-amazon/9.26.0", + "providers-amazon/9.26.0rc1", + "providers-google/21.2.0", + "providers-google/21.2.0rc1", + "providers-vespa/0.1.0", + "providers-microsoft-azure/13.1.2", + "providers-microsoft-azure/13.1.2rc1", +] + + +@pytest.mark.parametrize( + ("include_docs", "ref", "expected_providers", "expected_full_build", "log_substr"), + [ + # Standard wave dispatch: derives from git tags. + ( + "all-providers apache-airflow-providers", + "providers/2026-04-21", + "amazon google microsoft-azure vespa", + False, + "Derived wave providers (providers/2026-04-12 -> providers/2026-04-21)", + ), + # Meta-token but ref is RC-suffixed (not a wave tag). + ( + "all-providers apache-airflow-providers", + "providers/2026-04-21-rc1", + "", + True, + "not a wave-tag pattern", + ), + # Meta-token but ref is main. + ("all-providers", "main", "", True, "not a wave-tag pattern"), + # Meta-token, ref is sha. + ("apache-airflow-providers", "f8a258069f", "", True, "not a wave-tag pattern"), + # Explicit packages -- incremental path. + ("amazon google", "providers/2026-04-21", "amazon google", False, ""), + # Explicit with full-package-name token. + ( + "apache-airflow-providers-amazon", + "providers-amazon/9.27.0", + "amazon", + False, + "", + ), + # Explicit with dot-to-hyphen conversion. + ("common.ai", "any", "common-ai", False, ""), + # Explicit with non-provider tokens stripped. + ("apache-airflow helm-chart docker-stack", "any", "", False, ""), + # Explicit dedup. + ("amazon amazon google", "any", "amazon google", False, ""), + # Empty. + ("", "any", "", False, ""), + ], +) +def test_derive(include_docs, ref, expected_providers, expected_full_build, log_substr): + git = fake_git(wave_tags=WAVE_TAGS, provider_tags=WAVE_PROVIDER_TAGS) + providers, full_build, log = derive(include_docs, ref, git_runner=git) + assert providers == expected_providers + assert full_build is expected_full_build + if log_substr: + assert log_substr in log + + +def test_first_wave_no_predecessor(): + # Ref IS a wave tag but there's no predecessor in the list. + git = fake_git(wave_tags=["providers/2026-01-15"], provider_tags=[]) + providers, full_build, log = derive("all-providers", "providers/2026-01-15", git_runner=git) + assert providers == "" + assert full_build is True + assert "::warning::No predecessor wave tag found" in log + + +def test_wave_with_no_new_provider_tags(): + # Ref is a known wave tag with predecessor, but no per-provider tags + # between them (suspect: race with tag push, or no-op republish). + git = fake_git( + wave_tags=["providers/2026-05-19", "providers/2026-04-21"], + provider_tags=[], + ) + providers, full_build, log = derive("all-providers", "providers/2026-05-19", git_runner=git) + assert providers == "" + assert full_build is True + assert "::warning::Wave ref providers/2026-05-19 has no new per-provider tags" in log + + +def test_only_rc_tags_between_waves(): + # All per-provider tags between the two waves are RCs -- treated as no + # new finals, falls back to full rebuild with warning. + git = fake_git( + wave_tags=["providers/2026-05-19", "providers/2026-04-21"], + provider_tags=["providers-amazon/9.27.0rc1", "providers-google/21.3.0rc1"], + ) + providers, full_build, log = derive("all-providers", "providers/2026-05-19", git_runner=git) + assert providers == "" + assert full_build is True + assert "::warning::" in log + + +def test_meta_token_with_explicit_provider(): + # Meta-token wins -- explicit provider tokens are dropped. + git = fake_git(wave_tags=WAVE_TAGS, provider_tags=WAVE_PROVIDER_TAGS) + providers, full_build, log = derive("amazon all-providers", "providers/2026-04-21", git_runner=git) + assert providers == "amazon google microsoft-azure vespa" + assert full_build is False + + +def test_glob_matches_non_wave_tag_is_filtered_out(): + # The git glob `providers/[0-9]*-*-*` is broader than the strict + # `providers/YYYY-MM-DD` regex and matches release-with-date tags like + # `providers/2.11/2026-02-16` (Airflow 2.11 release on that date). + # Predecessor lookup must skip those so it picks the actual prior wave. + git = fake_git( + wave_tags=[ + "providers/2026-02-20", + "providers/2.11/2026-02-16", # noise: not a wave tag + "providers/2026-02-10", + ], + provider_tags=["providers-amazon/9.20.0"], + ) + providers, full_build, log = derive("all-providers", "providers/2026-02-20", git_runner=git) + # Predecessor must be the real prior wave (2026-02-10), not the noise. + assert providers == "amazon" + assert full_build is False + assert "providers/2026-02-10 -> providers/2026-02-20" in log diff --git a/uv.lock b/uv.lock index 935bae223c82c..d72bc25516114 100644 --- a/uv.lock +++ b/uv.lock @@ -7911,6 +7911,7 @@ name = "apache-airflow-registry-tools" version = "0.0.1" source = { editable = "dev/registry" } dependencies = [ + { name = "packaging" }, { name = "pydantic" }, { name = "pyyaml" }, { name = "tomli", marker = "python_full_version < '3.11'" }, @@ -7923,6 +7924,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "packaging", specifier = ">=24.0" }, { name = "pydantic", specifier = ">=2.12.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" },