diff --git a/.github/workflows/registry-build.yml b/.github/workflows/registry-build.yml index 27a1713629ca6..f3e0e8bc7f2db 100644 --- a/.github/workflows/registry-build.yml +++ b/.github/workflows/registry-build.yml @@ -59,6 +59,15 @@ jobs: timeout-minutes: 30 name: "Build & Publish Registry" runs-on: ubuntu-latest + env: + EXISTING_REGISTRY_DIR: /tmp/existing-registry + REGISTRY_DATA_DIR: dev/registry + REGISTRY_PROVIDERS_JSON: providers.json + REGISTRY_MODULES_JSON: modules.json + REGISTRY_SITE_DATA_DIR: registry/src/_data + REGISTRY_SITE_VERSIONS_DIR: registry/src/_data/versions + REGISTRY_SITE_LOGOS_DIR: registry/public/logos + REGISTRY_CACHE_CONTROL: public, max-age=300 permissions: contents: read id-token: write @@ -142,12 +151,15 @@ jobs: env: S3_BUCKET: ${{ steps.destination.outputs.bucket }} run: | - mkdir -p /tmp/existing-registry - PROVIDERS_URL="${S3_BUCKET}api/providers.json" - MODULES_URL="${S3_BUCKET}api/modules.json" - if aws s3 cp "${PROVIDERS_URL}" /tmp/existing-registry/providers.json 2>/dev/null; then + mkdir -p "${EXISTING_REGISTRY_DIR}" + PROVIDERS_URL="${S3_BUCKET}api/${REGISTRY_PROVIDERS_JSON}" + MODULES_URL="${S3_BUCKET}api/${REGISTRY_MODULES_JSON}" + if aws s3 cp \ + "${PROVIDERS_URL}" \ + "${EXISTING_REGISTRY_DIR}/${REGISTRY_PROVIDERS_JSON}" \ + 2>/dev/null; then echo "found=true" >> "${GITHUB_OUTPUT}" - aws s3 cp "${MODULES_URL}" /tmp/existing-registry/modules.json || true + aws s3 cp "${MODULES_URL}" "${EXISTING_REGISTRY_DIR}/${REGISTRY_MODULES_JSON}" || true else echo "found=false" >> "${GITHUB_OUTPUT}" echo "No existing registry data on S3" @@ -169,24 +181,28 @@ jobs: if: inputs.provider != '' && steps.download-existing.outputs.found == 'true' run: | uv run dev/registry/merge_registry_data.py \ - --existing-providers /tmp/existing-registry/providers.json \ - --existing-modules /tmp/existing-registry/modules.json \ - --new-providers dev/registry/providers.json \ - --new-modules dev/registry/modules.json \ - --output dev/registry/ + --existing-providers "${EXISTING_REGISTRY_DIR}/${REGISTRY_PROVIDERS_JSON}" \ + --existing-modules "${EXISTING_REGISTRY_DIR}/${REGISTRY_MODULES_JSON}" \ + --new-providers "${REGISTRY_DATA_DIR}/${REGISTRY_PROVIDERS_JSON}" \ + --new-modules "${REGISTRY_DATA_DIR}/${REGISTRY_MODULES_JSON}" \ + --output "${REGISTRY_DATA_DIR}/" - name: "Copy breeze output to registry data" run: | - mkdir -p registry/src/_data/versions - cp dev/registry/providers.json registry/src/_data/providers.json - cp dev/registry/modules.json registry/src/_data/modules.json - if [ -d dev/registry/output/versions ]; then - cp -r dev/registry/output/versions/* registry/src/_data/versions/ + mkdir -p "${REGISTRY_SITE_VERSIONS_DIR}" + cp \ + "${REGISTRY_DATA_DIR}/${REGISTRY_PROVIDERS_JSON}" \ + "${REGISTRY_SITE_DATA_DIR}/${REGISTRY_PROVIDERS_JSON}" + cp \ + "${REGISTRY_DATA_DIR}/${REGISTRY_MODULES_JSON}" \ + "${REGISTRY_SITE_DATA_DIR}/${REGISTRY_MODULES_JSON}" + if [ -d "${REGISTRY_DATA_DIR}/output/versions" ]; then + cp -r "${REGISTRY_DATA_DIR}/output/versions/"* "${REGISTRY_SITE_VERSIONS_DIR}/" fi # Copy provider logos extracted from providers/*/docs/integration-logos/ - if [ -d dev/registry/logos ]; then - mkdir -p registry/public/logos - cp -r dev/registry/logos/* registry/public/logos/ + if [ -d "${REGISTRY_DATA_DIR}/logos" ]; then + mkdir -p "${REGISTRY_SITE_LOGOS_DIR}" + cp -r "${REGISTRY_DATA_DIR}/logos/"* "${REGISTRY_SITE_LOGOS_DIR}/" fi - name: "Setup pnpm" @@ -224,7 +240,7 @@ jobs: S3_BUCKET: ${{ steps.destination.outputs.bucket }} run: | aws s3 sync registry/_site/ "${S3_BUCKET}" \ - --cache-control "public, max-age=300" + --cache-control "${REGISTRY_CACHE_CONTROL}" - name: "Publish version metadata" env: diff --git a/dev/registry/extract_metadata.py b/dev/registry/extract_metadata.py index 664e67f54e4a6..c1d2043d24d20 100644 --- a/dev/registry/extract_metadata.py +++ b/dev/registry/extract_metadata.py @@ -44,11 +44,24 @@ import tomllib import yaml +# External endpoints used by metadata extraction. +PYPISTATS_RECENT_URL = "https://pypistats.org/api/packages/{package_name}/recent" +PYPI_PACKAGE_JSON_URL = "https://pypi.org/pypi/{package_name}/json" +S3_DOC_URL = "http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com" +AIRFLOW_PROVIDER_DOCS_URL = "https://airflow.apache.org/docs/{package_name}/stable/" +AIRFLOW_PROVIDER_CONNECTIONS_URL = ( + "https://airflow.apache.org/docs/{package_name}/stable/connections/index.html" +) +AIRFLOW_PROVIDER_SOURCE_URL = ( + "https://github.com/apache/airflow/tree/providers-{provider_id}/{version}/providers/{provider_path}" +) +PYPI_PACKAGE_URL = "https://pypi.org/project/{package_name}/" + def fetch_pypi_downloads(package_name: str) -> dict[str, int]: """Fetch download statistics from pypistats.org API.""" try: - url = f"https://pypistats.org/api/packages/{package_name}/recent" + url = PYPISTATS_RECENT_URL.format(package_name=package_name) with urllib.request.urlopen(url, timeout=5) as response: data = json.loads(response.read().decode()) return { @@ -64,7 +77,7 @@ def fetch_pypi_downloads(package_name: str) -> dict[str, int]: def fetch_pypi_dates(package_name: str) -> dict[str, str]: """Fetch first release and latest release dates from PyPI JSON API.""" try: - url = f"https://pypi.org/pypi/{package_name}/json" + url = PYPI_PACKAGE_JSON_URL.format(package_name=package_name) with urllib.request.urlopen(url, timeout=10) as response: data = json.loads(response.read().decode()) @@ -109,7 +122,6 @@ def read_inventory(inv_path: Path) -> dict[str, str]: return result -S3_DOC_URL = "http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com" INVENTORY_CACHE_DIR = Path(__file__).parent / ".inventory_cache" INVENTORY_TTL = datetime.timedelta(hours=12) @@ -520,9 +532,7 @@ def main(): # Extract connection types from provider.yaml # Link to the connections index page since individual connection pages might not exist connection_types = [] - connections_index_url = ( - f"https://airflow.apache.org/docs/{package_name}/stable/connections/index.html" - ) + connections_index_url = AIRFLOW_PROVIDER_CONNECTIONS_URL.format(package_name=package_name) for conn in provider_yaml.get("connection-types", []): conn_type = conn.get("connection-type", "") hook_class = conn.get("hook-class-name", "") @@ -546,6 +556,7 @@ def main(): # Airflow version compatibility (from pyproject.toml dependencies) airflow_versions = determine_airflow_versions(pyproject_data["dependencies"]) + provider_source_path = provider_path.relative_to(PROVIDERS_DIR).as_posix() provider = Provider( id=provider_id, name=name, @@ -563,9 +574,11 @@ def main(): requires_python=pyproject_data["requires_python"], dependencies=pyproject_data["dependencies"], optional_extras=pyproject_data.get("optional_extras", {}), - docs_url=f"https://airflow.apache.org/docs/{package_name}/stable/", - source_url=f"https://github.com/apache/airflow/tree/providers-{provider_id}/{version}/providers/{provider_path.relative_to(PROVIDERS_DIR)}", - pypi_url=f"https://pypi.org/project/{package_name}/", + docs_url=AIRFLOW_PROVIDER_DOCS_URL.format(package_name=package_name), + source_url=AIRFLOW_PROVIDER_SOURCE_URL.format( + provider_id=provider_id, version=version, provider_path=provider_source_path + ), + pypi_url=PYPI_PACKAGE_URL.format(package_name=package_name), first_released=pypi_dates["first_released"], last_updated=pypi_dates["last_updated"], )