Move javascript compilation to host

.dockerignore

@@ -80,12 +80,8 @@
 
 # Git version is dynamically generated
 airflow/git_version
-# Exclude static www files generated by NPM
-airflow/www/static/coverage
-airflow/www/static/dist
+# Exclude mode_modules pulled by "yarn" for compilation of www files generated by NPM
 airflow/www/node_modules
-# Exclude static ui files generated by NPM
-airflow/ui/build
 airflow/ui/node_modules
 
 # Exclude link to docs
@@ -99,7 +95,7 @@ airflow/www/static/docs
 **/env/
 **/build/
 **/develop-eggs/
-**/dist/
+/dist/
 **/downloads/
 **/eggs/
 **/.eggs/

.github/boring-cyborg.yml

@@ -119,7 +119,6 @@ labelPRBasedOnFilePath:
     - airflow/www/.eslintrc
     - airflow/www/.stylelintignore
     - airflow/www/.stylelintrc
-    - airflow/www/compile_assets.sh
     - airflow/www/package.json
     - airflow/www/webpack.config.js
     - airflow/www/yarn.lock

.github/workflows/build-images.yml

@@ -233,6 +233,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           breeze static-checks --type update-providers-dependencies --all-files
           --show-diff-on-failure --color always || true
         if: needs.build-info.outputs.default-branch == 'main'
+      - name: Compile www assets
+        run: breeze compile-www-assets
       - name: >-
           Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
@@ -344,6 +346,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0
       - name: "Move dist packages to docker-context files"
         run: mv -v ./dist/*.whl ./docker-context-files
+      - name: Compile www assets
+        run: breeze compile-www-assets
       - name: >-
           Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.all-python-versions-list-as-string }}

.github/workflows/ci.yml

@@ -349,6 +349,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch == 'main'
+      - name: Compile www assets
+        run: breeze compile-www-assets
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
       - name: >
           Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
@@ -448,6 +453,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       - name: "Move dist packages to docker-context files"
         run: mv -v ./dist/*.whl ./docker-context-files
         if: needs.build-info.outputs.in-workflow-build == 'true'
+      - name: Compile www assets
+        run: breeze compile-www-assets
+        if: needs.build-info.outputs.in-workflow-build == 'true'
       - name: >
           Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.all-python-versions-list-as-string }}

.gitignore

@@ -13,7 +13,7 @@ unittests.db
 # Airflow temporary artifacts
 airflow/git_version
 airflow/www/static/coverage/
-airflow/www/static/dist
+airflow/www/*.log
 
 /logs/
 airflow-webserver.pid

.pre-commit-config.yaml

@@ -296,7 +296,7 @@ repos:
         exclude: ^airflow/_vendor/|^RELEASE_NOTES\.txt$|^airflow/www/static/css/material-icons\.css$|^images/.*$
         args:
           - --ignore-words=docs/spelling_wordlist.txt
-          - --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/pipeline_example.csv
+          - --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/pipeline_example.csv,airflow/www/*.log
           - --exclude-file=.codespellignorelines
   - repo: local
     hooks:
@@ -593,9 +593,20 @@ repos:
         name: stylelint
         entry: "stylelint"
         language: node
+        language_version: 18.6.0
         files: ^airflow/www/.*\.(css|scss|sass)$
         # Keep dependency versions in sync w/ airflow/www/package.json
         additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0']
+      - id: compile-www-assets
+        name: Compile www assets
+        language: node
+        language_version: 18.6.0
+        stages: ['manual']
+        'types_or': [javascript, tsx, ts]
+        files: ^airflow/www/
+        entry: ./scripts/ci/pre_commit/pre_commit_compile_www_assets.py
+        pass_filenames: false
+        additional_dependencies: ['yarn@1.22.19']
       - id: check-providers-init-file-missing
         name: Provider init file is missing
         pass_filenames: false
@@ -660,6 +671,7 @@ repos:
         description: Checks the style of Markdown files.
         entry: markdownlint
         language: node
+        language_version: 18.6.0
         types: [markdown]
         files: \.(md|mdown|markdown)$
         additional_dependencies: ['markdownlint-cli']
@@ -813,6 +825,24 @@ repos:
         additional_dependencies: ['rich>=12.4.4']
         pass_filenames: false
         files: ^tests/.*\.py$
+      - id: lint-javascript
+        name: ESLint against airflow/ui
+        language: node
+        language_version: 18.6.0
+        'types_or': [javascript, tsx, ts]
+        files: ^airflow/ui/
+        entry: ./scripts/ci/pre_commit/pre_commit_ui_lint.py
+        pass_filenames: false
+        additional_dependencies: ['yarn@1.22.19']
+      - id: lint-javascript
+        name: ESLint against current UI JavaScript files
+        language: node
+        language_version: 18.6.0
+        'types_or': [javascript, tsx, ts]
+        files: ^airflow/www/static/js/
+        entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py
+        additional_dependencies: ['yarn@1.22.19']
+        pass_filenames: false
         ## ADD MOST PRE-COMMITS ABOVE THAT LINE
         # The below pre-commits are those requiring CI image to be built
       - id: run-mypy
@@ -853,22 +883,6 @@ repos:
         pass_filenames: true
         exclude: ^airflow/_vendor/
         additional_dependencies: ['rich>=12.4.4', 'inputimeout']
-      - id: lint-javascript
-        name: ESLint against airflow/ui
-        language: python
-        'types_or': [javascript, tsx, ts]
-        files: ^airflow/ui/
-        entry: ./scripts/ci/pre_commit/pre_commit_ui_lint.py
-        pass_filenames: false
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout']
-      - id: lint-javascript
-        name: ESLint against current UI JavaScript files
-        language: python
-        'types_or': [javascript, tsx, ts]
-        files: ^airflow/www/static/js/
-        entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py
-        pass_filenames: false
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout']
       - id: update-migration-references
         name: Update migration ref doc
         language: python

BREEZE.rst

@@ -453,13 +453,30 @@ regenerate all those images (which might be needed in case new version of rich i
   :alt: Breeze regenerate-command-images
 
 
+Compiling www assets
+====================
+
+Airflow webserver needs to prepare www assets - compiled with node and yarn. The ``compile-www-assets``
+command takes care about it. This is needed when you want to run webserver inside of the breeze.
+
+.. image:: ./images/breeze/output-compile-www-assets.svg
+  :width: 100%
+  :alt: Breeze compile-www-assets
+
 Starting complete Airflow installation
 ======================================
 
 For testing Airflow oyou often want to start multiple components (in multiple terminals). Breeze has
 built-in ``start-airflow`` command that start breeze container, launches multiple terminals using tmux
 and launches all Airflow necessary components in those terminals.
 
+When you are starting airflow from local sources, www asset compilation is automatically executed before.
+
+.. code-block:: bash
+
+    breeze --python 3.7 --backend mysql start-airflow
+
+
 You can also use it to start any released version of Airflow from ``PyPI`` with the
 ``--use-airflow-version`` flag.
 
@@ -511,6 +528,7 @@ Those are commands mostly used by contributors:
 * Execute arbitrary command in the test environment with ``breeze shell`` command
 * Enter interactive shell in CI container when ``shell`` (or no command) is specified
 * Start containerised, development-friendly airflow installation with ``breeze start-airflow`` command
+* Compile www assets for webserver ``breeze compile-www-assets`` command
 * Build documentation with ``breeze build-docs`` command
 * Initialize local virtualenv with ``./scripts/tools/initialize_virtualenv.py`` command
 * Run static checks with autocomplete support ``breeze static-checks`` command
@@ -1590,24 +1608,14 @@ If you set these variables, next time when you enter the environment the new por
 Managing Dependencies
 ---------------------
 
-If you need to change apt dependencies in the ``Dockerfile.ci``, add Python packages in ``setup.py`` or
-add JavaScript dependencies in ``package.json``, you can either add dependencies temporarily for a single
-Breeze session or permanently in ``setup.py``, ``Dockerfile.ci``, or ``package.json`` files.
-
-Installing Dependencies for a Single Breeze Session
-...................................................
-
-You can install dependencies inside the container using ``sudo apt install``, ``pip install`` or
-``yarn install`` (in ``airflow/www`` folder) respectively. This is useful if you want to test something
-quickly while you are in the container. However, these changes are not retained: they disappear once you
-exit the container (except for the node.js dependencies if your sources are mounted to the container).
-Therefore, if you want to retain a new dependency, follow the second option described below.
+If you need to change apt dependencies in the ``Dockerfile.ci``, add Python packages in ``setup.py``
+for airflow and in provider.yaml for packages. If you add any "node" dependencies in ``airflow/www``
+or ``airflow/ui``, you need to compile them in the host with ``breeze compile-www-assets`` command.
 
 Adding Dependencies Permanently
 ...............................
 
-You can add dependencies to the ``Dockerfile.ci``, ``setup.py`` or ``package.json`` and rebuild the image.
-This should happen automatically if you modify any of these files.
+You can add dependencies to the ``Dockerfile.ci``, ``setup.py``.
 After you exit the container and re-run ``breeze``, Breeze detects changes in dependencies,
 asks you to confirm rebuilding the image and proceeds with rebuilding if you confirm (or skip it
 if you do not confirm). After rebuilding is done, Breeze drops you to shell. You may also use the

CONTRIBUTING.rst

@@ -1136,12 +1136,12 @@ itself comes bundled with jQuery and bootstrap. While they may be phased out
 over time, these packages are currently not managed with yarn.
 
 Make sure you are using recent versions of node and yarn. No problems have been
-found with node\>=8.11.3 and yarn\>=1.19.1.
+found with node\>=8.11.3 and yarn\>=1.19.1. The pre-commit framework of ours install
+node and yarn automatically when installed - if you use ``breeze`` you do not need to install
+neither node nor yarn.
 
-Installing yarn and its packages
---------------------------------
-
-Make sure yarn is available in your environment.
+Installing yarn and its packages manually
+-----------------------------------------
 
 To install yarn on macOS:
 
@@ -1165,27 +1165,6 @@ To install yarn on macOS:
     export PATH="$HOME/.yarn/bin:$PATH"
 
 4.  Install third-party libraries defined in ``package.json`` by running the
-    following commands within the ``airflow/www/`` directory:
-
-
-.. code-block:: bash
-
-    # from the root of the repository, move to where our JS package.json lives
-    cd airflow/www/
-    # run yarn install to fetch all the dependencies
-    yarn install
-
-
-These commands install the libraries in a new ``node_modules/`` folder within
-``www/``.
-
-Should you add or upgrade a node package, run
-``yarn add --dev <package>`` for packages needed in development or
-``yarn add <package>`` for packages used by the code.
-Then push the newly generated ``package.json`` and ``yarn.lock`` file so that we
-could get a reproducible build. See the `Yarn docs
-<https://yarnpkg.com/en/docs/cli/add#adding-dependencies->`_ for more details.
-
 
 Generate Bundled Files with yarn
 --------------------------------
@@ -1199,7 +1178,7 @@ commands:
     yarn run prod
 
     # Starts a web server that manages and updates your assets as you modify them
-    # You'll need to run the webserver in debug mode too: `airflow webserver -d`
+    # You'll need to run the webserver in debug mode too: ``airflow webserver -d``
     yarn run dev