diff --git a/.dockerignore b/.dockerignore index 73cf1cf88b936..1f9297738dda7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -80,12 +80,8 @@ # Git version is dynamically generated airflow/git_version -# Exclude static www files generated by NPM -airflow/www/static/coverage -airflow/www/static/dist +# Exclude mode_modules pulled by "yarn" for compilation of www files generated by NPM airflow/www/node_modules -# Exclude static ui files generated by NPM -airflow/ui/build airflow/ui/node_modules # Exclude link to docs @@ -99,7 +95,7 @@ airflow/www/static/docs **/env/ **/build/ **/develop-eggs/ -**/dist/ +/dist/ **/downloads/ **/eggs/ **/.eggs/ diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index e8ae68f971f09..e3e27e811b58f 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -119,7 +119,6 @@ labelPRBasedOnFilePath: - airflow/www/.eslintrc - airflow/www/.stylelintignore - airflow/www/.stylelintrc - - airflow/www/compile_assets.sh - airflow/www/package.json - airflow/www/webpack.config.js - airflow/www/yarn.lock diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index c9ebd3c1d0fa2..740db9fdd8ff4 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -233,6 +233,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze static-checks --type update-providers-dependencies --all-files --show-diff-on-failure --color always || true if: needs.build-info.outputs.default-branch == 'main' + - name: Compile www assets + run: breeze compile-www-assets - name: >- Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} @@ -344,6 +346,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files + - name: Compile www assets + run: breeze compile-www-assets - name: >- Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14fa44148e76e..8e03f22d535d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -349,6 +349,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: > needs.build-info.outputs.in-workflow-build == 'true' && needs.build-info.outputs.default-branch == 'main' + - name: Compile www assets + run: breeze compile-www-assets + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' - name: > Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} @@ -448,6 +453,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files if: needs.build-info.outputs.in-workflow-build == 'true' + - name: Compile www assets + run: breeze compile-www-assets + if: needs.build-info.outputs.in-workflow-build == 'true' - name: > Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.all-python-versions-list-as-string }} diff --git a/.gitignore b/.gitignore index b9de02c90e052..08f2692b286ac 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ unittests.db # Airflow temporary artifacts airflow/git_version airflow/www/static/coverage/ -airflow/www/static/dist +airflow/www/*.log /logs/ airflow-webserver.pid diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8ecb1c4562e9..c345601c4a88e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -296,7 +296,7 @@ repos: exclude: ^airflow/_vendor/|^RELEASE_NOTES\.txt$|^airflow/www/static/css/material-icons\.css$|^images/.*$ args: - --ignore-words=docs/spelling_wordlist.txt - - --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/pipeline_example.csv + - --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/pipeline_example.csv,airflow/www/*.log - --exclude-file=.codespellignorelines - repo: local hooks: @@ -593,9 +593,20 @@ repos: name: stylelint entry: "stylelint" language: node + language_version: 18.6.0 files: ^airflow/www/.*\.(css|scss|sass)$ # Keep dependency versions in sync w/ airflow/www/package.json additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0'] + - id: compile-www-assets + name: Compile www assets + language: node + language_version: 18.6.0 + stages: ['manual'] + 'types_or': [javascript, tsx, ts] + files: ^airflow/www/ + entry: ./scripts/ci/pre_commit/pre_commit_compile_www_assets.py + pass_filenames: false + additional_dependencies: ['yarn@1.22.19'] - id: check-providers-init-file-missing name: Provider init file is missing pass_filenames: false @@ -660,6 +671,7 @@ repos: description: Checks the style of Markdown files. entry: markdownlint language: node + language_version: 18.6.0 types: [markdown] files: \.(md|mdown|markdown)$ additional_dependencies: ['markdownlint-cli'] @@ -813,6 +825,24 @@ repos: additional_dependencies: ['rich>=12.4.4'] pass_filenames: false files: ^tests/.*\.py$ + - id: lint-javascript + name: ESLint against airflow/ui + language: node + language_version: 18.6.0 + 'types_or': [javascript, tsx, ts] + files: ^airflow/ui/ + entry: ./scripts/ci/pre_commit/pre_commit_ui_lint.py + pass_filenames: false + additional_dependencies: ['yarn@1.22.19'] + - id: lint-javascript + name: ESLint against current UI JavaScript files + language: node + language_version: 18.6.0 + 'types_or': [javascript, tsx, ts] + files: ^airflow/www/static/js/ + entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py + additional_dependencies: ['yarn@1.22.19'] + pass_filenames: false ## ADD MOST PRE-COMMITS ABOVE THAT LINE # The below pre-commits are those requiring CI image to be built - id: run-mypy @@ -853,22 +883,6 @@ repos: pass_filenames: true exclude: ^airflow/_vendor/ additional_dependencies: ['rich>=12.4.4', 'inputimeout'] - - id: lint-javascript - name: ESLint against airflow/ui - language: python - 'types_or': [javascript, tsx, ts] - files: ^airflow/ui/ - entry: ./scripts/ci/pre_commit/pre_commit_ui_lint.py - pass_filenames: false - additional_dependencies: ['rich>=12.4.4', 'inputimeout'] - - id: lint-javascript - name: ESLint against current UI JavaScript files - language: python - 'types_or': [javascript, tsx, ts] - files: ^airflow/www/static/js/ - entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py - pass_filenames: false - additional_dependencies: ['rich>=12.4.4', 'inputimeout'] - id: update-migration-references name: Update migration ref doc language: python diff --git a/BREEZE.rst b/BREEZE.rst index 5a1b03f14daa1..8ad8ba5c50140 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -453,6 +453,16 @@ regenerate all those images (which might be needed in case new version of rich i :alt: Breeze regenerate-command-images +Compiling www assets +==================== + +Airflow webserver needs to prepare www assets - compiled with node and yarn. The ``compile-www-assets`` +command takes care about it. This is needed when you want to run webserver inside of the breeze. + +.. image:: ./images/breeze/output-compile-www-assets.svg + :width: 100% + :alt: Breeze compile-www-assets + Starting complete Airflow installation ====================================== @@ -460,6 +470,13 @@ For testing Airflow oyou often want to start multiple components (in multiple te built-in ``start-airflow`` command that start breeze container, launches multiple terminals using tmux and launches all Airflow necessary components in those terminals. +When you are starting airflow from local sources, www asset compilation is automatically executed before. + +.. code-block:: bash + + breeze --python 3.7 --backend mysql start-airflow + + You can also use it to start any released version of Airflow from ``PyPI`` with the ``--use-airflow-version`` flag. @@ -511,6 +528,7 @@ Those are commands mostly used by contributors: * Execute arbitrary command in the test environment with ``breeze shell`` command * Enter interactive shell in CI container when ``shell`` (or no command) is specified * Start containerised, development-friendly airflow installation with ``breeze start-airflow`` command +* Compile www assets for webserver ``breeze compile-www-assets`` command * Build documentation with ``breeze build-docs`` command * Initialize local virtualenv with ``./scripts/tools/initialize_virtualenv.py`` command * Run static checks with autocomplete support ``breeze static-checks`` command @@ -1590,24 +1608,14 @@ If you set these variables, next time when you enter the environment the new por Managing Dependencies --------------------- -If you need to change apt dependencies in the ``Dockerfile.ci``, add Python packages in ``setup.py`` or -add JavaScript dependencies in ``package.json``, you can either add dependencies temporarily for a single -Breeze session or permanently in ``setup.py``, ``Dockerfile.ci``, or ``package.json`` files. - -Installing Dependencies for a Single Breeze Session -................................................... - -You can install dependencies inside the container using ``sudo apt install``, ``pip install`` or -``yarn install`` (in ``airflow/www`` folder) respectively. This is useful if you want to test something -quickly while you are in the container. However, these changes are not retained: they disappear once you -exit the container (except for the node.js dependencies if your sources are mounted to the container). -Therefore, if you want to retain a new dependency, follow the second option described below. +If you need to change apt dependencies in the ``Dockerfile.ci``, add Python packages in ``setup.py`` +for airflow and in provider.yaml for packages. If you add any "node" dependencies in ``airflow/www`` +or ``airflow/ui``, you need to compile them in the host with ``breeze compile-www-assets`` command. Adding Dependencies Permanently ............................... -You can add dependencies to the ``Dockerfile.ci``, ``setup.py`` or ``package.json`` and rebuild the image. -This should happen automatically if you modify any of these files. +You can add dependencies to the ``Dockerfile.ci``, ``setup.py``. After you exit the container and re-run ``breeze``, Breeze detects changes in dependencies, asks you to confirm rebuilding the image and proceeds with rebuilding if you confirm (or skip it if you do not confirm). After rebuilding is done, Breeze drops you to shell. You may also use the diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 323172ad31474..ed0c439e2bd9c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1136,12 +1136,12 @@ itself comes bundled with jQuery and bootstrap. While they may be phased out over time, these packages are currently not managed with yarn. Make sure you are using recent versions of node and yarn. No problems have been -found with node\>=8.11.3 and yarn\>=1.19.1. +found with node\>=8.11.3 and yarn\>=1.19.1. The pre-commit framework of ours install +node and yarn automatically when installed - if you use ``breeze`` you do not need to install +neither node nor yarn. -Installing yarn and its packages --------------------------------- - -Make sure yarn is available in your environment. +Installing yarn and its packages manually +----------------------------------------- To install yarn on macOS: @@ -1165,27 +1165,6 @@ To install yarn on macOS: export PATH="$HOME/.yarn/bin:$PATH" 4. Install third-party libraries defined in ``package.json`` by running the - following commands within the ``airflow/www/`` directory: - - -.. code-block:: bash - - # from the root of the repository, move to where our JS package.json lives - cd airflow/www/ - # run yarn install to fetch all the dependencies - yarn install - - -These commands install the libraries in a new ``node_modules/`` folder within -``www/``. - -Should you add or upgrade a node package, run -``yarn add --dev `` for packages needed in development or -``yarn add `` for packages used by the code. -Then push the newly generated ``package.json`` and ``yarn.lock`` file so that we -could get a reproducible build. See the `Yarn docs -`_ for more details. - Generate Bundled Files with yarn -------------------------------- @@ -1199,7 +1178,7 @@ commands: yarn run prod # Starts a web server that manages and updates your assets as you modify them - # You'll need to run the webserver in debug mode too: `airflow webserver -d` + # You'll need to run the webserver in debug mode too: ``airflow webserver -d`` yarn run dev diff --git a/Dockerfile b/Dockerfile index b4ddddef80ac1..7fd6b8bbf6138 100644 --- a/Dockerfile +++ b/Dockerfile @@ -397,101 +397,6 @@ function common::show_pip_version_and_location() { } EOF -# The content below is automatically copied from scripts/docker/prepare_node_modules.sh -COPY <<"EOF" /prepare_node_modules.sh -set -euo pipefail - -COLOR_BLUE=$'\e[34m' -readonly COLOR_BLUE -COLOR_RESET=$'\e[0m' -readonly COLOR_RESET - -function prepare_node_modules() { - echo - echo "${COLOR_BLUE}Preparing node modules${COLOR_RESET}" - echo - local www_dir - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then - # In case we are building from sources in production image, we should build the assets - www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www" - else - www_dir="$(python -m site --user-site)/airflow/www" - fi - pushd ${www_dir} || exit 1 - set +e - yarn install --frozen-lockfile --no-cache 2>/tmp/out-yarn-install.txt - local res=$? - if [[ ${res} != 0 ]]; then - >&2 echo - >&2 echo "Error when running yarn install:" - >&2 echo - >&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt - exit 1 - fi - rm -f /tmp/out-yarn-install.txt - popd || exit 1 -} - -prepare_node_modules -EOF - -# The content below is automatically copied from scripts/docker/compile_www_assets.sh -COPY <<"EOF" /compile_www_assets.sh -set -euo pipefail - -BUILD_TYPE=${BUILD_TYPE="prod"} -REMOVE_ARTIFACTS=${REMOVE_ARTIFACTS="true"} - -COLOR_BLUE=$'\e[34m' -readonly COLOR_BLUE -COLOR_RESET=$'\e[0m' -readonly COLOR_RESET - -function compile_www_assets() { - echo - echo "${COLOR_BLUE}Compiling www assets: running yarn ${BUILD_TYPE}${COLOR_RESET}" - echo - local www_dir - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then - # In case we are building from sources in production image, we should build the assets - www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www" - else - www_dir="$(python -m site --user-site)/airflow/www" - fi - pushd ${www_dir} || exit 1 - set +e - yarn run "${BUILD_TYPE}" 2>/tmp/out-yarn-run.txt - res=$? - if [[ ${res} != 0 ]]; then - >&2 echo - >&2 echo "Error when running yarn run:" - >&2 echo - >&2 cat /tmp/out-yarn-run.txt && rm -rf /tmp/out-yarn-run.txt - exit 1 - fi - rm -f /tmp/out-yarn-run.txt - set -e - local md5sum_file - md5sum_file="static/dist/sum.md5" - readonly md5sum_file - find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${md5sum_file}" - if [[ ${REMOVE_ARTIFACTS} == "true" ]]; then - echo - echo "${COLOR_BLUE}Removing generated node modules${COLOR_RESET}" - echo - rm -rf "${www_dir}/node_modules" - rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js} - else - echo - echo "${COLOR_BLUE}Leaving generated node modules${COLOR_RESET}" - echo - fi - popd || exit 1 -} - -compile_www_assets -EOF - # The content below is automatically copied from scripts/docker/pip COPY <<"EOF" /pip #!/usr/bin/env bash @@ -1120,7 +1025,6 @@ ARG DEV_APT_DEPS="\ libssl-dev \ locales \ lsb-release \ - nodejs \ openssh-client \ sasl2-bin \ software-properties-common \ @@ -1132,9 +1036,7 @@ ARG DEV_APT_DEPS="\ ARG ADDITIONAL_DEV_APT_DEPS="" ARG DEV_APT_COMMAND="\ - curl --silent --fail --location https://deb.nodesource.com/setup_14.x | \ - bash -o pipefail -o errexit -o nolog - \ - && curl --silent https://dl.yarnpkg.com/debian/pubkey.gpg | \ + curl --silent https://dl.yarnpkg.com/debian/pubkey.gpg | \ apt-key add - >/dev/null 2>&1\ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="echo" @@ -1196,18 +1098,10 @@ ARG INSTALL_PROVIDERS_FROM_SOURCES="false" # But it also can be `.` from local installation or GitHub URL pointing to specific branch or tag # Of Airflow. Note That for local source installation you need to have local sources of # Airflow checked out together with the Dockerfile and AIRFLOW_SOURCES_FROM and AIRFLOW_SOURCES_TO -# set to "." and "/opt/airflow" respectively. Similarly AIRFLOW_SOURCES_WWW_FROM/TO are set to right source -# and destination +# set to "." and "/opt/airflow" respectively. ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" # By default we do not upgrade to latest dependencies ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" -# By default we install latest airflow from PyPI so we do not need to copy sources of Airflow -# www to compile the assets but in case of breeze/CI builds we use latest sources and we override those -# those SOURCES_FROM/TO with "airflow/www" and "/opt/airflow/airflow/www" respectively. -# This is to rebuild the assets only when any of the www sources change -ARG AIRFLOW_SOURCES_WWW_FROM="Dockerfile" -ARG AIRFLOW_SOURCES_WWW_TO="/Dockerfile" - # By default we install latest airflow from PyPI so we do not need to copy sources of Airflow # but in case of breeze/CI builds we use latest sources and we override those # those SOURCES_FROM/TO with "." and "/opt/airflow" respectively @@ -1303,26 +1197,8 @@ RUN bash /scripts/docker/install_pip_version.sh; \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi -COPY --from=scripts compile_www_assets.sh prepare_node_modules.sh /scripts/docker/ -COPY --chown=airflow:0 ${AIRFLOW_SOURCES_WWW_FROM} ${AIRFLOW_SOURCES_WWW_TO} - -# hadolint ignore=SC2086, SC2010 -RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ - # only prepare node modules and compile assets if the prod image is build from sources - # otherwise they are already compiled-in the package. - bash /scripts/docker/prepare_node_modules.sh; \ - REMOVE_ARTIFACTS="true" BUILD_TYPE="prod" bash /scripts/docker/compile_www_assets.sh; \ - # Copy generated dist folder (otherwise it will be overridden by the COPY step below) - mv -f /opt/airflow/airflow/www/static/dist /tmp/dist; \ - fi; - COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} -# Copy back the generated dist folder -RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ - mv -f /tmp/dist /opt/airflow/airflow/www/static/dist; \ - fi; - # Add extra python dependencies ARG ADDITIONAL_PYTHON_DEPS="" diff --git a/Dockerfile.ci b/Dockerfile.ci index f95a5cedbf339..556b21ecc13d4 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -384,101 +384,6 @@ common::get_colors install_pipx_tools EOF -# The content below is automatically copied from scripts/docker/prepare_node_modules.sh -COPY <<"EOF" /prepare_node_modules.sh -set -euo pipefail - -COLOR_BLUE=$'\e[34m' -readonly COLOR_BLUE -COLOR_RESET=$'\e[0m' -readonly COLOR_RESET - -function prepare_node_modules() { - echo - echo "${COLOR_BLUE}Preparing node modules${COLOR_RESET}" - echo - local www_dir - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then - # In case we are building from sources in production image, we should build the assets - www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www" - else - www_dir="$(python -m site --user-site)/airflow/www" - fi - pushd ${www_dir} || exit 1 - set +e - yarn install --frozen-lockfile --no-cache 2>/tmp/out-yarn-install.txt - local res=$? - if [[ ${res} != 0 ]]; then - >&2 echo - >&2 echo "Error when running yarn install:" - >&2 echo - >&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt - exit 1 - fi - rm -f /tmp/out-yarn-install.txt - popd || exit 1 -} - -prepare_node_modules -EOF - -# The content below is automatically copied from scripts/docker/compile_www_assets.sh -COPY <<"EOF" /compile_www_assets.sh -set -euo pipefail - -BUILD_TYPE=${BUILD_TYPE="prod"} -REMOVE_ARTIFACTS=${REMOVE_ARTIFACTS="true"} - -COLOR_BLUE=$'\e[34m' -readonly COLOR_BLUE -COLOR_RESET=$'\e[0m' -readonly COLOR_RESET - -function compile_www_assets() { - echo - echo "${COLOR_BLUE}Compiling www assets: running yarn ${BUILD_TYPE}${COLOR_RESET}" - echo - local www_dir - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then - # In case we are building from sources in production image, we should build the assets - www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www" - else - www_dir="$(python -m site --user-site)/airflow/www" - fi - pushd ${www_dir} || exit 1 - set +e - yarn run "${BUILD_TYPE}" 2>/tmp/out-yarn-run.txt - res=$? - if [[ ${res} != 0 ]]; then - >&2 echo - >&2 echo "Error when running yarn run:" - >&2 echo - >&2 cat /tmp/out-yarn-run.txt && rm -rf /tmp/out-yarn-run.txt - exit 1 - fi - rm -f /tmp/out-yarn-run.txt - set -e - local md5sum_file - md5sum_file="static/dist/sum.md5" - readonly md5sum_file - find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${md5sum_file}" - if [[ ${REMOVE_ARTIFACTS} == "true" ]]; then - echo - echo "${COLOR_BLUE}Removing generated node modules${COLOR_RESET}" - echo - rm -rf "${www_dir}/node_modules" - rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js} - else - echo - echo "${COLOR_BLUE}Leaving generated node modules${COLOR_RESET}" - echo - fi - popd || exit 1 -} - -compile_www_assets -EOF - # The content below is automatically copied from scripts/docker/install_airflow.sh COPY <<"EOF" /install_airflow.sh @@ -663,11 +568,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then echo echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}" echo - if [[ -d "${AIRFLOW_SOURCES}/airflow/www/" ]]; then - pushd "${AIRFLOW_SOURCES}/airflow/www/" >/dev/null - ./ask_for_recompile_assets_if_needed.sh - popd >/dev/null - fi # Cleanup the logs, tmp when entering the environment sudo rm -rf "${AIRFLOW_SOURCES}"/logs/* sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/* @@ -795,6 +695,8 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then touch /usr/lib/google-cloud-sdk/bin/gcloud ln -s -f /usr/bin/gcloud /usr/lib/google-cloud-sdk/bin/gcloud + in_container_fix_ownership + if [[ ${SKIP_SSH_SETUP="false"} == "false" ]]; then # Set up ssh keys echo 'yes' | ssh-keygen -t rsa -C your_email@youremail.com -m PEM -P '' -f ~/.ssh/id_rsa \ @@ -1025,16 +927,6 @@ COPY <<"EOF" /entrypoint_exec.sh exec /bin/bash "${@}" EOF -############################################################################################## -# This is the www image where we keep all inlined files needed to build ui -# It is copied separately to volume to speed up building and avoid cache miss on changed -# file permissions. -# We use PYTHON_BASE_IMAGE to make sure that the scripts are different for different platforms. -############################################################################################## -FROM ${PYTHON_BASE_IMAGE} as www -COPY airflow/www/package.json airflow/www/yarn.lock airflow/www/webpack.config.js / -COPY airflow/www/static/ /static - FROM ${PYTHON_BASE_IMAGE} as main # Nolog bash flag is currently ignored - but you can replace it with other flags (for example @@ -1060,8 +952,7 @@ RUN echo "Base image version: ${PYTHON_BASE_IMAGE}" ARG ADDITIONAL_DEV_APT_DEPS="" ARG DEV_APT_COMMAND="\ - curl --silent --fail --location https://deb.nodesource.com/setup_14.x | bash - \ - && curl --silent --fail https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - >/dev/null 2>&1 \ + curl --silent --fail https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - >/dev/null 2>&1 \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_ENV_VARS="" @@ -1104,7 +995,6 @@ RUN apt-get update \ "${DISTRO_LIBENCHANT}" \ locales \ netcat \ - nodejs \ rsync \ sasl2-bin \ sudo \ @@ -1306,25 +1196,6 @@ COPY --from=scripts install_pipx_tools.sh /scripts/docker/ # dependencies installed in Airflow RUN bash /scripts/docker/install_pipx_tools.sh -# Copy package.json and yarn.lock to install node modules -# this way even if other static check files change, node modules will not need to be installed -# we want to keep node_modules so we can do this step separately from compiling assets -COPY --from=www package.json yarn.lock ${AIRFLOW_SOURCES}/airflow/www/ -COPY --from=scripts prepare_node_modules.sh /scripts/docker/ - -# Package JS/css for production -RUN bash /scripts/docker/prepare_node_modules.sh - -# Copy all the needed www/ for assets compilation. Done as two separate COPY -# commands so as otherwise it copies the _contents_ of static/ in to www/ -COPY --from=www webpack.config.js ${AIRFLOW_SOURCES}/airflow/www/ -COPY --from=www static ${AIRFLOW_SOURCES}/airflow/www/static/ -COPY --from=scripts compile_www_assets.sh /scripts/docker/ - -# Build artifacts without removing temporary artifacts (we will need them for incremental changes) -# in build mode -RUN REMOVE_ARTIFACTS="false" BUILD_TYPE="build" bash /scripts/docker/compile_www_assets.sh - # Airflow sources change frequently but dependency configuration won't change that often # We copy setup.py and other files needed to perform setup of dependencies # So in case setup.py changes we can install latest dependencies required. diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 1b18aa659d685..6934919b952b2 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -215,6 +215,8 @@ require Breeze Docker image to be build locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | codespell | Run codespell to check for common misspellings in files | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ +| compile-www-assets | Compile www assets | | ++--------------------------------------------------------+------------------------------------------------------------------+---------+ | create-missing-init-py-files-tests | Create missing init.py files in tests | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ | debug-statements | Detect accidentally committed debug statements | | @@ -252,7 +254,7 @@ require Breeze Docker image to be build locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | lint-helm-chart | Lint Helm Chart | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| lint-javascript | * ESLint against airflow/ui | * | +| lint-javascript | * ESLint against airflow/ui | | | | * ESLint against current UI JavaScript files | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ | lint-json-schema | * Lint JSON Schema files with JSON Schema | | diff --git a/airflow/www/ask_for_recompile_assets_if_needed.sh b/airflow/www/ask_for_recompile_assets_if_needed.sh deleted file mode 100755 index 8342ae7631bf2..0000000000000 --- a/airflow/www/ask_for_recompile_assets_if_needed.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -e - -if [[ ${GITHUB_ACTIONS:="false"} == "true" ]]; then - echo - echo -e "${BLUE}Skipping asset compilation check in CI.${NO_COLOR}" - echo - exit 0 -fi - -cd "$( dirname "${BASH_SOURCE[0]}" )" - -MD5SUM_FILE="static/dist/sum.md5" -readonly MD5SUM_FILE - -GREEN='\033[1;32m' -YELLOW='\033[1;33m' -NO_COLOR='\033[0m' - -md5sum=$(find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum) -old_md5sum=$(cat "${MD5SUM_FILE}" 2>/dev/null || true) -if [[ ${old_md5sum} != "${md5sum}" ]]; then - if [[ ( ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ) && ${USE_AIRFLOW_VERSION:=} == "" ]]; then - echo - echo -e "${YELLOW}Recompiling assets as they have changed and you need them for 'start_airflow' command${NO_COLOR}" - echo - ./compile_assets.sh - else - echo - echo -e "${YELLOW}WARNING: It seems that the generated assets files do not match the content of the sources.${NO_COLOR}" - echo "To recompile assets, run:" - echo "" - echo " ./airflow/www/compile_assets.sh" - echo "" - fi -else - echo - echo -e "${GREEN}No need for www assets recompilation.${NO_COLOR}" - echo -fi diff --git a/breeze-legacy b/breeze-legacy index 0bb257494c915..b03f5ae18f441 100755 --- a/breeze-legacy +++ b/breeze-legacy @@ -136,14 +136,6 @@ function breeze::setup_default_breeze_constants() { AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/opt/airflow"} export AIRFLOW_SOURCES_TO - # Sources by default are installed from local sources when using breeze - AIRFLOW_SOURCES_WWW_FROM=${AIRFLOW_SOURCES_WWW_FROM:="./airflow/www"} - export AIRFLOW_SOURCES_WWW_FROM - - # They are copied to /opt/airflow by default in breeze - AIRFLOW_SOURCES_WWW_TO=${AIRFLOW_SOURCES_WWW_TO:="/opt/airflow/airflow/www"} - export AIRFLOW_SOURCES_WWW_TO - # Unlike in CI scripts, in breeze by default production image is installed from sources export AIRFLOW_INSTALLATION_METHOD="." diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 268dad01a8c94..404c2e2863d30 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -75,6 +75,7 @@ assert_pre_commit_installed, filter_out_none, run_command, + run_compile_www_assets, ) from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE @@ -83,6 +84,7 @@ "commands": [ "shell", "start-airflow", + "compile-www-assets", "exec", "stop", "build-docs", @@ -357,6 +359,8 @@ def start_airflow( extra_args: Tuple, ): """Enter breeze.py environment and starts all Airflow components in the tmux session.""" + if use_airflow_version is None: + run_compile_www_assets(verbose=verbose, dry_run=dry_run) enter_shell( verbose=verbose, dry_run=dry_run, @@ -546,6 +550,26 @@ def static_checks( sys.exit(static_checks_result.returncode) +@main.command( + name="compile-www-assets", + help="Compiles www assets.", + context_settings=dict( + ignore_unknown_options=True, + allow_extra_args=True, + ), +) +@option_verbose +@option_dry_run +def compile_www_assets( + verbose: bool, + dry_run: bool, +): + compile_www_assets_result = run_compile_www_assets(verbose=verbose, dry_run=dry_run) + if compile_www_assets_result.returncode != 0: + get_console().print("[warn]New assets were generated[/]") + sys.exit(0) + + @main.command(name="stop", help="Stop running breeze environment.") @option_verbose @option_dry_run diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 6a2c331e6d706..87dae0c9a2e79 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -65,7 +65,7 @@ ) from airflow_breeze.utils.parallel import check_async_run_results from airflow_breeze.utils.python_versions import get_python_version_list -from airflow_breeze.utils.run_utils import RunCommandResult, run_command +from airflow_breeze.utils.run_utils import RunCommandResult, run_command, run_compile_www_assets RELEASE_MANAGEMENT_PARAMETERS = { "breeze prepare-airflow-package": [ @@ -234,6 +234,7 @@ def prepare_airflow_packages( debug: bool, ): perform_environment_checks(verbose=verbose) + run_compile_www_assets(verbose=verbose, dry_run=dry_run) shell_params = ShellParams( verbose=verbose, github_repository=github_repository, diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 6e2914a2d9157..dba556b3310e3 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -225,18 +225,12 @@ def get_airflow_extras(): 'setup.cfg', 'Dockerfile.ci', '.dockerignore', - 'scripts/docker/compile_www_assets.sh', 'scripts/docker/common.sh', 'scripts/docker/install_additional_dependencies.sh', 'scripts/docker/install_airflow.sh', 'scripts/docker/install_airflow_dependencies_from_branch_tip.sh', 'scripts/docker/install_from_docker_context_files.sh', 'scripts/docker/install_mysql.sh', - 'airflow/www/package.json', - 'airflow/www/yarn.lock', - 'airflow/www/webpack.config.js', - 'airflow/ui/package.json', - 'airflow/ui/yarn.lock', ] ENABLED_SYSTEMS = "" @@ -265,8 +259,6 @@ def get_airflow_extras(): AIRFLOW_SOURCES_FROM = "." AIRFLOW_SOURCES_TO = "/opt/airflow" -AIRFLOW_SOURCES_WWW_FROM = "./airflow/www" -AIRFLOW_SOURCES_WWW_TO = "/opt/airflow/airflow/www" DEFAULT_EXTRAS = [ # BEGINNING OF EXTRAS LIST UPDATED BY PRE COMMIT diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 86a25b6999e08..9e30b95f69c8d 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -26,8 +26,6 @@ from airflow_breeze.global_constants import ( AIRFLOW_SOURCES_FROM, AIRFLOW_SOURCES_TO, - AIRFLOW_SOURCES_WWW_FROM, - AIRFLOW_SOURCES_WWW_TO, get_airflow_extras, get_airflow_version, ) @@ -73,10 +71,6 @@ def args_for_remote_install(self) -> List: build_args = [] build_args.extend( [ - "--build-arg", - "AIRFLOW_SOURCES_WWW_FROM=empty", - "--build-arg", - "AIRFLOW_SOURCES_WWW_TO=/empty", "--build-arg", "AIRFLOW_SOURCES_FROM=empty", "--build-arg", @@ -159,10 +153,6 @@ def extra_docker_build_flags(self) -> List[str]: "--build-arg", f"AIRFLOW_SOURCES_TO={AIRFLOW_SOURCES_TO}", "--build-arg", - f"AIRFLOW_SOURCES_WWW_FROM={AIRFLOW_SOURCES_WWW_FROM}", - "--build-arg", - f"AIRFLOW_SOURCES_WWW_TO={AIRFLOW_SOURCES_WWW_TO}", - "--build-arg", f"AIRFLOW_INSTALLATION_METHOD={self.installation_method}", "--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}", diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 4b36d48e33cd1..35e0a1ba7514c 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -64,6 +64,7 @@ 'check-system-tests-tocs', 'check-xml', 'codespell', + 'compile-www-assets', 'create-missing-init-py-files-tests', 'debug-statements', 'detect-private-key', diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 29268be8b3c92..36c5c6fa937a8 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -354,3 +354,34 @@ def get_runnable_ci_image(verbose: bool, dry_run: bool) -> str: instruction=f"breeze build-image --python {python_version}", ) return airflow_image + + +def run_compile_www_assets( + verbose: bool, + dry_run: bool, +): + from airflow_breeze.utils.docker_command_utils import perform_environment_checks + + assert_pre_commit_installed(verbose=verbose) + perform_environment_checks(verbose=verbose) + command_to_execute = [ + sys.executable, + "-m", + "pre_commit", + 'run', + "--hook-stage", + "manual", + 'compile-www-assets', + '--all-files', + ] + env = os.environ.copy() + compile_www_assets_result = run_command( + command_to_execute, + verbose=verbose, + dry_run=dry_run, + check=False, + no_output_dump_on_exception=True, + text=True, + env=env, + ) + return compile_www_assets_result diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index a1e4f1d9f0da5..9aa2f25544d7a 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -208,16 +208,6 @@ You can see some examples of those in: | | | "/opt/airflow" when you install Airflow | | | | from local sources. | +------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_SOURCES_WWW_FROM`` | ``Dockerfile`` | Sources of Airflow WWW files used for | -| | | asset compilation. Set it to | -| | | "./airflow/www" when | -| | | you install Airflow from local sources | -+------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_SOURCES_WWW_TO`` | ``/Dockerfile`` | Target for Airflow files used for | -| | | asset compilation. Set it to | -| | | "/opt/airflow/airflow/www" when | -| | | you install Airflow from local sources. | -+------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_VERSION_SPECIFICATION`` | | Optional - might be used for using limit | | | | for Airflow version installation - for | | | | example ``<2.0.2`` for automated builds. | diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 2271f6eca7c28..b84eb16be8da9 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -8,6 +8,7 @@ build-image:b62509a59badf3aa230e4562df751002 build-prod-image:b9aa5dcefcd7067a6698f68c05adc2b8 cleanup:9a94bd1063296ea86e895f671db0b330 command-hash-export:83bc6a4a8c60b62da3d0f00e81d2c3ea +compile-www-assets:5f0d8af0a2a074f2ee577604662598c8 config:92653afc11889e1b78e3a2e38f41107f docker-compose-tests:8ae3b6211fd31db81a750d1c6b96ec3d exec:e4329909b8b2a610fa4fad5116c4b896 @@ -28,7 +29,7 @@ self-upgrade:b5437c0a1a91533a11ee9d0a9692369c setup-autocomplete:355b72dee171c2fcba46fc90ac7c97b0 shell:4680295fdd8a276d51518d29360c365c start-airflow:92cf775a952439a32d409cd2536da507 -static-checks:8b4c0a3891e1d65ddf832ec4f5b71491 +static-checks:6d29e34384ef071a6eea88cfcc2edb93 stop:8ebd8a42f1003495d37b884de5ac7ce6 tests:e39111ecbd33a65ababb3cbfbac2b069 verify-image:a6b3c70957aea96a5d4d261f23359a2d diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 2e21ac9804afa..77e487bfd108f 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Breeze commands + Breeze commands - + - - -Usage: breeze [OPTIONS] COMMAND [ARGS]... - -╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│available in dist folder respectively. Implies --mount-sources `remove`.                │ -│(none | wheel | sdist | <airflow_version>)                                              │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ -│entering breeze.                                                                        │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =    │ -│selected).                                                                              │ -│(selected | all | skip | remove)                                                        │ -│[default: selected]                                                                     │ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│shell          Enter breeze.py environment. this is the default command use when no other is selected.            │ -│start-airflow  Enter breeze.py environment and starts all Airflow components in the tmux session.                 │ -│exec           Joins the interactive shell of running airflow container                                           │ -│stop           Stop running breeze environment.                                                                   │ -│build-docs     Build documentation in the container.                                                              │ -│static-checks  Run static checks.                                                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│docker-compose-tests Run docker-compose tests.                                                                    │ -│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ -│cleanup                  Cleans the cache of parameters, docker cache and optionally - currently downloaded       │ -│images.                                                                                  │ -│self-upgrade             Self upgrade Breeze.                                                                     │ -│setup-autocomplete       Enables autocompletion of breeze commands.                                               │ -│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ -│regenerate-command-imagesRegenerate breeze command images.                                                        │ -│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ -│images should be regenerated).                                                           │ -│version                  Print information about version of apache-airflow-breeze.                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-image   Build CI image. Include building multiple images for all python versions (sequentially).            │ -│pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                │ -│verify-image  Verify CI image.                                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-prod-image Build Production image. Include building multiple images for all or selected Python versions     │ -│sequentially.                                                                                    │ -│pull-prod-image  Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ -│verify-prod-imageVerify Production image.                                                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI commands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│fix-ownership                  Fix ownership of source files to be same as host user.                             │ -│free-space                     Free space for jobs run in CI.                                                     │ -│resource-check                 Check if available docker resources are enough.                                    │ -│selective-check                Checks what kind of tests should be run for an incoming commit.                    │ -│find-newer-dependencies        Finds which dependencies are being upgraded.                                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze [OPTIONS] COMMAND [ARGS]... + +╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ +│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ +│available in dist folder respectively. Implies --mount-sources `remove`.                │ +│(none | wheel | sdist | <airflow_version>)                                              │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ +│entering breeze.                                                                        │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ +│--force-buildForce image build no matter if it is determined as needed.│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =    │ +│selected).                                                                              │ +│(selected | all | skip | remove)                                                        │ +│[default: selected]                                                                     │ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│shell               Enter breeze.py environment. this is the default command use when no other is selected.       │ +│start-airflow       Enter breeze.py environment and starts all Airflow components in the tmux session.            │ +│compile-www-assets  Compiles www assets.                                                                          │ +│exec                Joins the interactive shell of running airflow container                                      │ +│stop                Stop running breeze environment.                                                              │ +│build-docs          Build documentation in the container.                                                         │ +│static-checks       Run static checks.                                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│docker-compose-tests Run docker-compose tests.                                                                    │ +│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ +│cleanup                  Cleans the cache of parameters, docker cache and optionally - currently downloaded       │ +│images.                                                                                  │ +│self-upgrade             Self upgrade Breeze.                                                                     │ +│setup-autocomplete       Enables autocompletion of breeze commands.                                               │ +│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ +│regenerate-command-imagesRegenerate breeze command images.                                                        │ +│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ +│images should be regenerated).                                                           │ +│version                  Print information about version of apache-airflow-breeze.                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build-image   Build CI image. Include building multiple images for all python versions (sequentially).            │ +│pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                │ +│verify-image  Verify CI image.                                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build-prod-image Build Production image. Include building multiple images for all or selected Python versions     │ +│sequentially.                                                                                    │ +│pull-prod-image  Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ +│verify-prod-imageVerify Production image.                                                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI commands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│fix-ownership                  Fix ownership of source files to be same as host user.                             │ +│free-space                     Free space for jobs run in CI.                                                     │ +│resource-check                 Check if available docker resources are enough.                                    │ +│selective-check                Checks what kind of tests should be run for an incoming commit.                    │ +│find-newer-dependencies        Finds which dependencies are being upgraded.                                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ +│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ +│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ +│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ +│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ +│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-compile-www-assets.svg b/images/breeze/output-compile-www-assets.svg new file mode 100644 index 0000000000000..5ffd5d828f2aa --- /dev/null +++ b/images/breeze/output-compile-www-assets.svg @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: compile-www-assets + + + + + + + + + + +Usage: breeze compile-www-assets [OPTIONS] + +Compiles www assets. + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output-static-checks.svg b/images/breeze/output-static-checks.svg index 1a1d0b17ac32e..5b4254e0bb228 100644 --- a/images/breeze/output-static-checks.svg +++ b/images/breeze/output-static-checks.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: static-checks + Command: static-checks - + - - -Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... - -Run static checks. - -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--type-tType(s) of the static checks to run (multiple can be added).                            │ -│(all | black | blacken-docs | check-airflow-2-2-compatibility |                         │ -│check-airflow-config-yaml-consistent | check-apache-license-rat |                       │ -│check-base-operator-partial-arguments | check-base-operator-usage |                     │ -│check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ -│check-builtin-literals | check-changelog-has-no-duplicates |                            │ -│check-daysago-import-from-utils | check-docstring-param-types | check-example-dags-urls │ -│| check-executables-have-shebangs | check-extra-packages-references | check-extras-order│ -│| check-for-inclusive-language | check-hooks-apply | check-incorrect-use-of-LoggingMixin│ -│| check-integrations-are-consistent | check-lazy-logging | check-merge-conflict |       │ -│check-newsfragments-are-valid | check-no-providers-in-core-examples |                   │ -│check-no-relative-imports | check-persist-credentials-disabled-in-github-workflows |    │ -│check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ -│check-provider-yaml-valid | check-providers-init-file-missing |                         │ -│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ -│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ -│check-start-date-not-used-in-defaults | check-system-tests-present |                    │ -│check-system-tests-tocs | check-xml | codespell | create-missing-init-py-files-tests |  │ -│debug-statements | detect-private-key | doctoc | end-of-file-fixer | fix-encoding-pragma│ -│| flynt | identity | insert-license | isort | lint-chart-schema | lint-css |            │ -│lint-dockerfile | lint-helm-chart | lint-javascript | lint-json-schema | lint-markdown |│ -│lint-openapi | mixed-line-ending | pretty-format-json | pydocstyle | python-no-log-warn │ -│| pyupgrade | replace-bad-characters | rst-backticks | run-flake8 | run-mypy |          │ -│run-shellcheck | static-check-autoflake | trailing-whitespace | update-breeze-cmd-output│ -│| update-breeze-readme-config-hash | update-extras | update-in-the-wild-to-be-sorted |  │ -│update-inlined-dockerfile-scripts | update-local-yml-file | update-migration-references │ -│| update-providers-dependencies | update-setup-cfg-file |                               │ -│update-spelling-wordlist-to-be-sorted | update-supported-versions |                     │ -│update-vendored-in-k8s-json-schema | update-version | yamllint | yesqa)                 │ -│--file-fList of files to run the checks on.(PATH)│ -│--all-files-aRun checks on all files.│ -│--show-diff-on-failure-sShow diff for files modified by the checks.│ -│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually  │ -│exclusive with --last-commit.                                                              │ -│(TEXT)                                                                                     │ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... + +Run static checks. + +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--type-tType(s) of the static checks to run (multiple can be added).                            │ +│(all | black | blacken-docs | check-airflow-2-2-compatibility |                         │ +│check-airflow-config-yaml-consistent | check-apache-license-rat |                       │ +│check-base-operator-partial-arguments | check-base-operator-usage |                     │ +│check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ +│check-builtin-literals | check-changelog-has-no-duplicates |                            │ +│check-daysago-import-from-utils | check-docstring-param-types | check-example-dags-urls │ +│| check-executables-have-shebangs | check-extra-packages-references | check-extras-order│ +│| check-for-inclusive-language | check-hooks-apply | check-incorrect-use-of-LoggingMixin│ +│| check-integrations-are-consistent | check-lazy-logging | check-merge-conflict |       │ +│check-newsfragments-are-valid | check-no-providers-in-core-examples |                   │ +│check-no-relative-imports | check-persist-credentials-disabled-in-github-workflows |    │ +│check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ +│check-provider-yaml-valid | check-providers-init-file-missing |                         │ +│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ +│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ +│check-start-date-not-used-in-defaults | check-system-tests-present |                    │ +│check-system-tests-tocs | check-xml | codespell | compile-www-assets |                  │ +│create-missing-init-py-files-tests | debug-statements | detect-private-key | doctoc |   │ +│end-of-file-fixer | fix-encoding-pragma | flynt | identity | insert-license | isort |   │ +│lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart | lint-javascript |    │ +│lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | pretty-format-json│ +│| pydocstyle | python-no-log-warn | pyupgrade | replace-bad-characters | rst-backticks |│ +│run-flake8 | run-mypy | run-shellcheck | static-check-autoflake | trailing-whitespace | │ +│update-breeze-cmd-output | update-breeze-readme-config-hash | update-extras |           │ +│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |                   │ +│update-local-yml-file | update-migration-references | update-providers-dependencies |   │ +│update-setup-cfg-file | update-spelling-wordlist-to-be-sorted |                         │ +│update-supported-versions | update-vendored-in-k8s-json-schema | update-version |       │ +│yamllint | yesqa)                                                                       │ +│--file-fList of files to run the checks on.(PATH)│ +│--all-files-aRun checks on all files.│ +│--show-diff-on-failure-sShow diff for files modified by the checks.│ +│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually  │ +│exclusive with --last-commit.                                                              │ +│(TEXT)                                                                                     │ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index a5292e4befaae..6916e6b9eb75c 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -217,26 +217,12 @@ function initialization::initialize_available_integrations() { FILES_FOR_REBUILD_CHECK=() # Determine which files trigger rebuild check -# -# !!!!!!!!!! IMPORTANT NOTE !!!!!!!!!! -# When you add files here, please make sure to not add files -# with the same name. And if you do - make sure that files with the -# same name are stored in directories with different name. For -# example we have two package.json files here, but they are in -# directories with different names (`www` and `ui`). -# The problem is that md5 hashes of those files are stored in -# `./build/directory` in the same directory as -.md5sum. -# For example md5sum of the `airflow/www/package.json` file is stored -# as `www-package.json` and `airflow/ui/package.json` as `ui-package.json`, -# The file list here changes extremely rarely. -# !!!!!!!!!! IMPORTANT NOTE !!!!!!!!!! function initialization::initialize_files_for_rebuild_check() { FILES_FOR_REBUILD_CHECK+=( "setup.py" "setup.cfg" "Dockerfile.ci" ".dockerignore" - "scripts/docker/compile_www_assets.sh" "scripts/docker/common.sh" "scripts/docker/install_additional_dependencies.sh" "scripts/docker/install_airflow.sh" @@ -244,11 +230,6 @@ function initialization::initialize_files_for_rebuild_check() { "scripts/docker/install_from_docker_context_files.sh" "scripts/docker/install_mysql.sh" "scripts/docker/install_postgres.sh" - "airflow/www/package.json" - "airflow/www/yarn.lock" - "airflow/www/webpack.config.js" - "airflow/ui/package.json" - "airflow/ui/yarn.lock" ) } @@ -440,13 +421,6 @@ function initialization::initialize_image_build_variables() { AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/Dockerfile"} export AIRFLOW_SOURCES_TO - # By default no sources are copied to image - AIRFLOW_SOURCES_WWW_FROM=${AIRFLOW_SOURCES_WWW_FROM:="Dockerfile"} - export AIRFLOW_SOURCES_WWW_FROM - - AIRFLOW_SOURCES_WWW_TO=${AIRFLOW_SOURCES_WWW_TO:="/Dockerfile"} - export AIRFLOW_SOURCES_WWW_TO - # By default in scripts production docker image is installed from PyPI package export AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD:="apache-airflow"} @@ -735,8 +709,6 @@ Production image build variables: AIRFLOW_VERSION_SPECIFICATION: '${AIRFLOW_VERSION_SPECIFICATION}' AIRFLOW_SOURCES_FROM: '${AIRFLOW_SOURCES_FROM}' AIRFLOW_SOURCES_TO: '${AIRFLOW_SOURCES_TO}' - AIRFLOW_SOURCES_WWW_FROM: '${AIRFLOW_SOURCES_WWW_FROM}' - AIRFLOW_SOURCES_WWW_TO: '${AIRFLOW_SOURCES_WWW_TO}' Detected GitHub environment: diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py new file mode 100755 index 0000000000000..c7dfae99364c4 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import subprocess +from pathlib import Path + +if __name__ not in ("__main__", "__mp_main__"): + raise SystemExit( + "This file is intended to be executed as an executable program. You cannot use it as a module." + f"To run this script, run the ./{__file__} command" + ) + +if __name__ == '__main__': + dir = Path("airflow") / "www" + subprocess.check_call(['yarn', 'install', '--frozen-lockfile'], cwd=str(dir)) + subprocess.check_call(['yarn', 'run', 'build'], cwd=str(dir)) diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py index eb8084285b14f..6cffea919bea7 100755 --- a/scripts/ci/pre_commit/pre_commit_ui_lint.py +++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py @@ -15,8 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import os -import sys +import subprocess from pathlib import Path if __name__ not in ("__main__", "__mp_main__"): @@ -25,38 +24,7 @@ f"To run this script, run the ./{__file__} command" ) -AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() -# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like -VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" -DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" - if __name__ == '__main__': - sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) - from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags - from airflow_breeze.utils.path_utils import create_static_check_volumes - from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - - airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) - create_static_check_volumes() - cmd_result = run_command( - [ - "docker", - "run", - "-t", - *get_extra_docker_flags(MOUNT_SELECTED), - "-e", - "SKIP_ENVIRONMENT_INITIALIZATION=true", - "-e", - "PRINT_INFO_FROM_SCRIPTS=false", - "--pull", - "never", - airflow_image, - "-c", - 'cd airflow/ui && yarn --frozen-lockfile --non-interactive && yarn run lint', - ], - check=False, - verbose=VERBOSE, - dry_run=DRY_RUN, - ) - sys.exit(cmd_result.returncode) + dir = Path("airflow") / "ui" + subprocess.check_call(['yarn', '--frozen-lockfile', '--non-interactive'], cwd=str(dir)) + subprocess.check_call(['yarn', 'run', 'lint'], cwd=str(dir)) diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py index da8dc63426ac4..9b77ef56fb880 100755 --- a/scripts/ci/pre_commit/pre_commit_www_lint.py +++ b/scripts/ci/pre_commit/pre_commit_www_lint.py @@ -15,8 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import os -import sys +import subprocess from pathlib import Path if __name__ not in ("__main__", "__mp_main__"): @@ -25,39 +24,7 @@ f"To run this script, run the ./{__file__} command" ) -AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() -GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") -# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like -VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" -DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" - if __name__ == '__main__': - sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) - from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags - from airflow_breeze.utils.path_utils import create_static_check_volumes - from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - - airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) - create_static_check_volumes() - cmd_result = run_command( - [ - "docker", - "run", - "-t", - *get_extra_docker_flags(MOUNT_SELECTED), - "-e", - "SKIP_ENVIRONMENT_INITIALIZATION=true", - "-e", - "PRINT_INFO_FROM_SCRIPTS=false", - "--pull", - "never", - airflow_image, - "-c", - 'cd airflow/www && yarn --frozen-lockfile --non-interactive && yarn run lint', - ], - check=False, - verbose=VERBOSE, - dry_run=DRY_RUN, - ) - sys.exit(cmd_result.returncode) + dir = Path("airflow") / "www" + subprocess.check_call(['yarn', '--frozen-lockfile', '--non-interactive'], cwd=dir) + subprocess.check_call(['yarn', 'run', 'lint'], cwd=dir) diff --git a/scripts/docker/compile_www_assets.sh b/scripts/docker/compile_www_assets.sh deleted file mode 100644 index 7bc87d85e9db0..0000000000000 --- a/scripts/docker/compile_www_assets.sh +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck shell=bash disable=SC2086 -set -euo pipefail - -BUILD_TYPE=${BUILD_TYPE="prod"} -REMOVE_ARTIFACTS=${REMOVE_ARTIFACTS="true"} - -COLOR_BLUE=$'\e[34m' -readonly COLOR_BLUE -COLOR_RESET=$'\e[0m' -readonly COLOR_RESET - -# Installs additional dependencies passed as Argument to the Docker build command -function compile_www_assets() { - echo - echo "${COLOR_BLUE}Compiling www assets: running yarn ${BUILD_TYPE}${COLOR_RESET}" - echo - local www_dir - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then - # In case we are building from sources in production image, we should build the assets - www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www" - else - www_dir="$(python -m site --user-site)/airflow/www" - fi - pushd ${www_dir} || exit 1 - set +e - yarn run "${BUILD_TYPE}" 2>/tmp/out-yarn-run.txt - res=$? - if [[ ${res} != 0 ]]; then - >&2 echo - >&2 echo "Error when running yarn run:" - >&2 echo - >&2 cat /tmp/out-yarn-run.txt && rm -rf /tmp/out-yarn-run.txt - exit 1 - fi - rm -f /tmp/out-yarn-run.txt - set -e - local md5sum_file - md5sum_file="static/dist/sum.md5" - readonly md5sum_file - find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${md5sum_file}" - if [[ ${REMOVE_ARTIFACTS} == "true" ]]; then - echo - echo "${COLOR_BLUE}Removing generated node modules${COLOR_RESET}" - echo - rm -rf "${www_dir}/node_modules" - rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js} - else - echo - echo "${COLOR_BLUE}Leaving generated node modules${COLOR_RESET}" - echo - fi - popd || exit 1 -} - -compile_www_assets diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index ae4961ea649e2..ef6d092779543 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -71,11 +71,6 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then echo echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}" echo - if [[ -d "${AIRFLOW_SOURCES}/airflow/www/" ]]; then - pushd "${AIRFLOW_SOURCES}/airflow/www/" >/dev/null - ./ask_for_recompile_assets_if_needed.sh - popd >/dev/null - fi # Cleanup the logs, tmp when entering the environment sudo rm -rf "${AIRFLOW_SOURCES}"/logs/* sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/* @@ -203,6 +198,8 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then touch /usr/lib/google-cloud-sdk/bin/gcloud ln -s -f /usr/bin/gcloud /usr/lib/google-cloud-sdk/bin/gcloud + in_container_fix_ownership + if [[ ${SKIP_SSH_SETUP="false"} == "false" ]]; then # Set up ssh keys echo 'yes' | ssh-keygen -t rsa -C your_email@youremail.com -m PEM -P '' -f ~/.ssh/id_rsa \ diff --git a/scripts/docker/prepare_node_modules.sh b/scripts/docker/prepare_node_modules.sh deleted file mode 100644 index 6c39de9c4ddb0..0000000000000 --- a/scripts/docker/prepare_node_modules.sh +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck shell=bash disable=SC2086 -set -euo pipefail - -COLOR_BLUE=$'\e[34m' -readonly COLOR_BLUE -COLOR_RESET=$'\e[0m' -readonly COLOR_RESET - -# Prepares node modules needed to compile WWW assets -function prepare_node_modules() { - echo - echo "${COLOR_BLUE}Preparing node modules${COLOR_RESET}" - echo - local www_dir - if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then - # In case we are building from sources in production image, we should build the assets - www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www" - else - www_dir="$(python -m site --user-site)/airflow/www" - fi - pushd ${www_dir} || exit 1 - set +e - yarn install --frozen-lockfile --no-cache 2>/tmp/out-yarn-install.txt - local res=$? - if [[ ${res} != 0 ]]; then - >&2 echo - >&2 echo "Error when running yarn install:" - >&2 echo - >&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt - exit 1 - fi - rm -f /tmp/out-yarn-install.txt - popd || exit 1 -} - -prepare_node_modules diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 33a7b67d280fa..f313457dd4a1b 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -127,23 +127,21 @@ function in_container_fix_ownership() { DIRECTORIES_TO_FIX=( "/dist" "/files" - "/root/.aws" - "/root/.azure" - "/root/.config/gcloud" - "/root/.docker" "/opt/airflow/logs" "/opt/airflow/docs" "/opt/airflow/dags" - "${AIRFLOW_SOURCES}" + "/opt/airflow/airflow/" ) - echo - echo "${COLOR_BLUE}Fixing ownership of generated files as Host OS is ${HOST_OS}${COLOR_RESET}" - echo "${COLOR_BLUE}Directories: ${DIRECTORIES_TO_FIX[*]}${COLOR_RESET}" - echo - find "${DIRECTORIES_TO_FIX[@]}" -print0 -user root 2>/dev/null | - xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1 - echo "${COLOR_BLUE}Fixed ownership of generated files." - echo + count_matching=$(find "${DIRECTORIES_TO_FIX[@]}" -mindepth 1 -user root -printf . 2>/dev/null | wc -m || true) + if [[ ${count_matching=} != "0" && ${count_matching=} != "" ]]; then + echo + echo "${COLOR_BLUE}Fixing ownership of ${count_matching} root owned files on ${HOST_OS}${COLOR_RESET}" + echo + find "${DIRECTORIES_TO_FIX[@]}" -mindepth 1 -user root -print0 2> /dev/null | + xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1 + echo "${COLOR_BLUE}Fixed ownership of generated files${COLOR_RESET}." + echo + fi else echo echo "${COLOR_YELLOW}Skip fixing ownership of generated files as Host OS is ${HOST_OS}${COLOR_RESET}" diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh index 791724d2bbdc4..50d4fb8f8ac10 100755 --- a/scripts/in_container/run_prepare_airflow_packages.sh +++ b/scripts/in_container/run_prepare_airflow_packages.sh @@ -63,7 +63,7 @@ function prepare_airflow_packages() { fi # Prepare airflow's wheel - PYTHONUNBUFFERED=1 python setup.py compile_assets "${tag_build[@]}" "${packages[@]}" + PYTHONUNBUFFERED=1 python setup.py "${tag_build[@]}" "${packages[@]}" # clean-up rm -rf -- *egg-info* diff --git a/airflow/www/compile_assets.sh b/tests/internal_api/__init__.py old mode 100755 new mode 100644 similarity index 67% rename from airflow/www/compile_assets.sh rename to tests/internal_api/__init__.py index ba4576614d8cc..13a83393a9124 --- a/airflow/www/compile_assets.sh +++ b/tests/internal_api/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,20 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -set -e - -cd "$( dirname "${BASH_SOURCE[0]}" )" - -MD5SUM_FILE="static/dist/sum.md5" -readonly MD5SUM_FILE - -# first bump up package.json manually, commit and tag -if [[ -d ./static/dist ]]; then - rm -f ./static/dist/* -fi - -yarn install --frozen-lockfile -yarn run build - -find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${MD5SUM_FILE}"