Make options output of breeze command consistent and complete

.github/workflows/ci.yml

@@ -426,8 +426,9 @@ jobs:
           cache: 'pip'
           cache-dependency-path: ./dev/breeze/setup*
       - run: python -m pip install --editable ./dev/breeze/
-      - run: python -m pytest ./dev/breeze/ -n auto --color=yes
       - run: breeze setup version
+      - run: python -m pytest ./dev/breeze/ -n auto --color=yes
+      - run: breeze setup check-all-params-in-groups
 
   tests-www:
     timeout-minutes: 10

BREEZE.rst

@@ -328,15 +328,6 @@ so you can change it at any place, and run
 
 inside container, to enable modified tmux configurations.
 
-.. raw:: html
-
-    <div align="center">
-      <a href="https://youtu.be/4MCTXq-oF68?t=78">
-        <img src="images/breeze/overlayed_breeze_installation.png" width="640"
-             alt="Airflow Breeze - Installation">
-      </a>
-    </div>
-
 Regular development tasks
 =========================
 
@@ -369,15 +360,6 @@ default settings.
 You can see which value of the parameters that can be stored persistently in cache marked with >VALUE<
 in the help of the commands.
 
-.. raw:: html
-
-    <div align="center">
-      <a href="https://youtu.be/4MCTXq-oF68?t=389">
-        <img src="images/breeze/overlayed_breeze_select_backend_python.png" width="640"
-             alt="Airflow Breeze - Selecting Python and Backend version">
-      </a>
-    </div>
-
 Building the documentation
 --------------------------
 
@@ -422,15 +404,6 @@ Those are all available flags of ``build-docs`` command:
   :alt: Breeze build documentation
 
 
-.. raw:: html
-
-    <div align="center">
-      <a href="https://youtu.be/4MCTXq-oF68?t=1760">
-        <img src="images/breeze/overlayed_breeze_build_docs.png" width="640"
-             alt="Airflow Breeze - Build docs">
-      </a>
-    </div>
-
 Running static checks
 ---------------------
 
@@ -1380,6 +1353,18 @@ check if there are any images that need regeneration.
   :width: 100%
   :alt: Breeze setup regenerate-command-images
 
+Breeze check-all-params-in-groups
+...................
+
+When you add a breeze command or modify a parameter, you are also supposed to make sure that "rich groups"
+for the command is present and that all parameters are assigned to the right group so they can be
+nicely presented in ``--help`` output. You can check that via ``check-all-params-in-groups`` command.
+
+.. image:: ./images/breeze/output_setup_check-all-params-in-groups.svg
+  :target: https://github.com/apache/airflow/main/images/breeze/output_setup_check-all-params-in-groups.svg
+  :width: 100%
+  :alt: Breeze setup check-all-params-in-group
+
 
 CI tasks
 --------

dev/breeze/src/airflow_breeze/commands/ci_commands_config.py

@@ -36,6 +36,7 @@
             ],
         }
     ],
+    "breeze ci free-space": [],
     "breeze ci selective-check": [
         {
             "name": "Selective check flags",
@@ -69,4 +70,5 @@
             ],
         }
     ],
+    "breeze ci resource-check": [],
 }

dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

@@ -255,12 +255,11 @@ def run_build(ci_image_params: BuildCiParams) -> None:
 @option_verify
 @option_wait_for_image
 @option_image_tag_for_pulling
-@option_include_success_outputs
 @option_tag_as_latest
 @option_github_repository
 @option_verbose
 @option_dry_run
-@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
+@click.argument("extra-pytest-args", nargs=-1, type=click.UNPROCESSED)
 def pull(
     python: str,
     run_in_parallel: bool,

dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py

@@ -74,7 +74,6 @@
                 "--github-token",
                 "--github-username",
                 "--platform",
-                "--login-to-github-registry",
                 "--push",
                 "--empty-image",
                 "--prepare-buildx-cache",

dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py

@@ -81,7 +81,6 @@
                 "--run-in-parallel",
                 "--parallelism",
                 "--python-versions",
-                "--kubernetes-versions",
                 "--skip-cleanup",
                 "--debug-resources",
                 "--include-success-outputs",
@@ -115,7 +114,6 @@
             "options": [
                 "--python",
                 "--kubernetes-version",
-                "--image-tag",
             ],
         },
         {
@@ -217,8 +215,6 @@
             "name": "K8S cluster creation flags",
             "options": [
                 "--force-recreate-cluster",
-                "--forwarded-port-number",
-                "--api-server-port",
             ],
         },
         {

dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py

@@ -93,7 +93,6 @@
                 "--github-token",
                 "--github-username",
                 "--platform",
-                "--login-to-github-registry",
                 "--push",
                 "--empty-image",
                 "--prepare-buildx-cache",

dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py

@@ -73,6 +73,7 @@
             "options": [
                 "--debug",
                 "--github-repository",
+                "--base-branch",
             ],
         }
     ],
@@ -95,7 +96,6 @@
                 "--python-versions",
                 "--skip-cleanup",
                 "--debug-resources",
-                "--include-success-outputs",
             ],
         },
     ],
@@ -112,4 +112,16 @@
             ],
         }
     ],
+    "breeze release-management generate-issue-content": [
+        {
+            "name": "Generate issue content flags",
+            "options": [
+                "--github-token",
+                "--suffix",
+                "--only-available-in-dist",
+                "--excluded-pr-list",
+                "--disable-progress",
+            ],
+        }
+    ],
 }

dev/breeze/src/airflow_breeze/commands/setup_commands.py

@@ -473,6 +473,108 @@ def regenerate_help_images_for_all_commands(commands: tuple[str, ...], check_onl
     return 1
 
 
+COMMON_PARAM_NAMES = ["--help", "--verbose", "--dry-run", "--answer"]
+COMMAND_PATH_PREFIX = "dev/breeze/src/airflow_breeze/commands/"
+
+
+def command_path(command: str) -> str:
+    return COMMAND_PATH_PREFIX + command.replace("-", "_") + "_commands.py"
+
+
+def command_path_config(command: str) -> str:
+    return COMMAND_PATH_PREFIX + command.replace("-", "_") + "_commands_config.py"
+
+
+def find_options_in_options_list(option: str, option_list: list[list[str]]) -> int | None:
+    for i, options in enumerate(option_list):
+        if option in options:
+            return i
+    return None
+
+
+def check_params(command: str, subcommand: str | None, command_dict: dict[str, Any]) -> bool:
+    import rich_click
+
+    get_console().print(
+        f"[info]Checking if params are in groups for specified command :{command}"
+        + (f" {subcommand}." if subcommand else ".")
+    )
+    errors_detected = False
+    options = rich_click.rich_click.OPTION_GROUPS
+    rich_click_key = "breeze " + command + (f" {subcommand}" if subcommand else "")
+    if rich_click_key not in options:
+        get_console().print(
+            f"[error]The command `{rich_click_key}` not found in dictionaries "
+            f"defined in rich click configuration."
+        )
+        get_console().print(f"[warning]Please add it to the `{command_path_config(command)}`.")
+        return False
+    rich_click_param_groups = options[rich_click_key]
+    defined_param_names = [
+        param["opts"] for param in command_dict["params"] if param["param_type_name"] == "option"
+    ]
+    for group in rich_click_param_groups:
+        if "options" in group:
+            for param in group["options"]:
+                index = find_options_in_options_list(param, defined_param_names)
+                if index is not None:
+                    del defined_param_names[index]
+                else:
+                    get_console().print(
+                        f"[error]Parameter `{param}` is not defined as option in {command_path(command)} in "
+                        f"`{rich_click_key}`, but is present in the "
+                        f"`{rich_click_key}` group in `{command_path_config(command)}`."
+                    )
+                    get_console().print(
+                        "[warning]Please remove it from there od add parameter in "
+                        "the command. NOTE! This error might be printed when the option is"
+                        "added twice in the command definition!."
+                    )
+                    errors_detected = True
+    for param in COMMON_PARAM_NAMES:
+        index = find_options_in_options_list(param, defined_param_names)
+        if index is not None:
+            del defined_param_names[index]
+    if defined_param_names:
+        for param in defined_param_names:
+            get_console().print(
+                f"[error]Parameter `{param}` is defined in `{command_path(command)}` in "
+                f"`{rich_click_key}`, but does not belong to any group options "
+                f"in `{rich_click_key}` group in `{command_path_config(command)}` and is not common."
+            )
+            get_console().print("[warning]Please add it to relevant group or create new group there.")
+        errors_detected = True
+    return errors_detected
+
+
+def check_that_all_params_are_in_groups(commands: tuple[str, ...]) -> int:
+    Console(width=int(SCREENSHOT_WIDTH), color_system="standard")
+    env = os.environ.copy()
+    env["AIRFLOW_SOURCES_ROOT"] = str(AIRFLOW_SOURCES_ROOT)
+    env["RECORD_BREEZE_WIDTH"] = SCREENSHOT_WIDTH
+    env["TERM"] = "xterm-256color"
+    env["PYTHONPATH"] = str(BREEZE_SOURCES_DIR)
+    with Context(main) as ctx:
+        the_context_dict = ctx.to_info_dict()
+    commands_dict = the_context_dict["command"]["commands"]
+    if commands:
+        commands_list = list(commands)
+    else:
+        commands_list = commands_dict.keys()
+    errors_detected = False
+    for command in commands_list:
+        current_command_dict = commands_dict[command]
+        if "commands" in current_command_dict:
+            subcommands = current_command_dict["commands"]
+            for subcommand in sorted(subcommands.keys()):
+                if check_params(command, subcommand, subcommands[subcommand]):
+                    errors_detected = True
+        else:
+            if check_params(command, None, current_command_dict):
+                errors_detected = True
+    return 1 if errors_detected else 0
+
+
 @setup.command(name="regenerate-command-images", help="Regenerate breeze command images.")
 @click.option("--force", is_flag=True, help="Forces regeneration of all images", envvar="FORCE")
 @click.option(
@@ -496,3 +598,18 @@ def regenerate_command_images(command: tuple[str, ...], force: bool, check_only:
         commands=command, check_only=check_only, force=force
     )
     sys.exit(return_code)
+
+
+@setup.command(name="check-all-params-in-groups", help="Check that all parameters are put in groups.")
+@click.option(
+    "--command",
+    help="Command(s) to regenerate images for (optional, might be repeated)",
+    show_default=True,
+    multiple=True,
+    type=BetterChoice(get_commands()),
+)
+@option_verbose
+@option_dry_run
+def check_all_params_in_groups(command: tuple[str, ...]):
+    return_code = check_that_all_params_are_in_groups(commands=command)
+    sys.exit(return_code)

dev/breeze/src/airflow_breeze/commands/setup_commands_config.py

@@ -29,6 +29,14 @@
     ],
 }
 SETUP_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
+    "breeze setup check-all-params-in-groups": [
+        {
+            "name": "Check all params in groups flags",
+            "options": [
+                "--command",
+            ],
+        }
+    ],
     "breeze setup self-upgrade": [
         {
             "name": "Self-upgrade flags",
@@ -70,4 +78,5 @@
             ],
         },
     ],
+    "breeze setup version": [],
 }

dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

@@ -75,6 +75,7 @@
             "options": [
                 "--image-tag",
                 "--mount-sources",
+                "--skip-provider-tests",
             ],
         },
     ],

dev/breeze/src/airflow_breeze/utils/custom_param_types.py

@@ -112,6 +112,8 @@ class VerboseOption(ParamType):
     Stores and allows to retrieve verbose option
     """
 
+    name = "VerboseOption"
+
     def convert(self, value, param, ctx):
         set_verbose(coerce_bool_value(value))
         return super().convert(value, param, ctx)
@@ -122,6 +124,8 @@ class DryRunOption(ParamType):
     Stores and allows to retrieve dry_run option
     """
 
+    name = "DryRunOption"
+
     def convert(self, value, param, ctx):
         set_dry_run(coerce_bool_value(value))
         return super().convert(value, param, ctx)

dev/breeze/tests/test_commands.py

@@ -47,3 +47,9 @@ def test_get_extra_docker_flags_skip():
     flags = get_extra_docker_flags(MOUNT_SKIP)
     assert "/empty," not in "".join(flags)
     assert len(flags) < 10
+
+
+def test_get_no_unknown_common_options():
+    flags = get_extra_docker_flags(MOUNT_SKIP)
+    assert "/empty," not in "".join(flags)
+    assert len(flags) < 10

images/breeze/output-commands-hash.txt

@@ -11,9 +11,9 @@ ci:resource-check:1d4fe47dff9fc64ac1648ec4beb2d85c
 ci:selective-check:3a085894f24cb909812fbc8253a21e13
 ci:9884bf4cef3e70270068de89b9d5df2f
 ci-image:build:f8d0fafe7246ca7ac412df8c520908a3
-ci-image:pull:8d67d5ff546d99178497d50e38f5eb22
+ci-image:pull:c16c6e57c748bfe9b365b4ffafb18472
 ci-image:verify:aee88f55e8837028d19316356e29b009
-ci-image:6ea662b02888efd868fa919c2c704cd1
+ci-image:0c727b84a764e09c54e461b30ad0410b
 cleanup:231de69d5f47ba29c883164e4575e310
 compile-www-assets:c8a8c4f002f7246d0541897fc7c70313
 exec:42bbd3c1659128b0341ae118c3482da2
@@ -44,11 +44,12 @@ release-management:release-prod-images:c9bc40938e0efad49e51ef66e83f9527
 release-management:verify-provider-packages:88bd609aff6d09d52ab8d80d6e055e7b
 release-management:07fecd3ed21621993c55ac2487a412ce
 setup:autocomplete:03343478bf1d0cf9c101d454cdb63b68
+setup:check-all-params-in-groups:af8367c0efadadbef80844672e62dd1a
 setup:config:3ffcd35dd24b486ddf1d08b797e3d017
-setup:regenerate-command-images:ab2d83c339fa3a42b0c819b6b6cc88ae
+setup:regenerate-command-images:d18a745ea79cecf05dd84bfc3920c5c4
 setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa
 setup:version:123b462a421884dc2320ffc5e54b2478
-setup:fbabee281b69f818091d780b24bd815a
+setup:edf5b96953fb28b3440a9a228b134910
 shell:76e0f530b7af514a2aad3032b6516c46
 start-airflow:06d4aeb5f1b65f6b975f3f915558d0b3
 static-checks:d20293484c293d721728713350a23b00