chore: fix tests and enable CI

.ci/Dockerfile.cypress

@@ -5,7 +5,8 @@ WORKDIR $APP
 
 COPY package.json yarn.lock .yarnrc $APP/
 COPY viz-lib $APP/viz-lib
-RUN npm install yarn@1.22.22 -g && yarn --frozen-lockfile --network-concurrency 1 > /dev/null
+RUN --mount=type=secret,id=npmrc,target=/root/.npmrc,mode=0644 \
+    npm install yarn@1.22.22 -g && yarn --frozen-lockfile --network-concurrency 1 > /dev/null
 
 COPY . $APP
 

.ci/compose.cypress.yaml

@@ -4,11 +4,14 @@ x-redash-service: &redash-service
     args:
       install_groups: "main"
       code_coverage: ${CODE_COVERAGE}
+    secrets:
+      - npmrc
 x-redash-environment: &redash-environment
   REDASH_LOG_LEVEL: "INFO"
   REDASH_REDIS_URL: "redis://redis:6379/0"
   POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb"
   REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres"
+  ASSETDB_DATABASE_URI: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres"
   REDASH_RATELIMIT_ENABLED: "false"
   REDASH_ENFORCE_CSRF: "true"
   REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF"
@@ -44,6 +47,8 @@ services:
     build:
       context: ../
       dockerfile: .ci/Dockerfile.cypress
+      secrets:
+        - npmrc
     depends_on:
       - server
       - worker
@@ -71,3 +76,6 @@ services:
     restart: unless-stopped
     environment:
       POSTGRES_HOST_AUTH_METHOD: "trust"
+secrets:
+  npmrc:
+    file: ${HOME}/.npmrc

.github/CODEOWNERS

@@ -1 +1 @@
-* @stacklet/platform
+* @stacklet/engineering

.github/composites/install/action.yml

@@ -0,0 +1,48 @@
+name: Install Dependencies
+description: AWS Auth and Install Dependencies
+inputs:
+  aws-role:
+    description: AWS Credential Access Role
+    required: false
+  aws-region:
+    description: AWS Region
+    default: us-east-1
+  install-yarn-deps:
+    description: Whether to install yarn dependencies
+    default: 'true'
+
+runs:
+  using: composite
+  steps:
+    - uses: wistia/parse-tool-versions@32f568a4ffd4bfa7720ebf93f171597d1ebc979a # v2.1.1
+      with:
+        postfix: _TOOL_VERSION
+
+    - uses: extractions/setup-crate@4993624604c307fbca528d28a3c8b60fa5ecc859 # v1.4.0
+      with:
+        repo: casey/just
+        version: ${{ env.JUST_TOOL_VERSION }}
+
+    - uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
+      if: inputs.install-yarn-deps == 'true'
+      with:
+        node-version: ${{ env.NODEJS_TOOL_VERSION }}
+        cache: 'yarn'
+
+    - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+      if: inputs.install-yarn-deps == 'true'
+      with:
+        role-to-assume: ${{ inputs.aws-role }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Login to AWS CodeArtifact
+      if: inputs.install-yarn-deps == 'true'
+      shell: bash
+      run: just pkg-login
+
+    - name: Install Dependencies
+      if: inputs.install-yarn-deps == 'true'
+      shell: bash
+      run: |
+        npm install --verbose --global --force yarn@${{ env.YARN_TOOL_VERSION }}
+        yarn cache clean && yarn --frozen-lockfile --network-concurrency 1

.github/workflows/ci.yml

@@ -2,176 +2,106 @@ name: Tests
 on:
   push:
     branches:
-      - master
+      - stacklet/integration
   pull_request:
     branches:
-      - master
-env:
-  NODE_VERSION: 18
-  YARN_VERSION: 1.22.22
+      - stacklet/integration
+permissions:
+  contents: read
 jobs:
   backend-lint:
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
     steps:
-      - if: github.event.pull_request.mergeable == 'false'
-        name: Exit if PR is not mergeable
-        run: exit 1
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
-          fetch-depth: 1
-          ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-python@v5
+          persist-credentials: false
+      - name: Install Dependencies
+        uses: ./.github/composites/install
+        with:
+          install-yarn-deps: 'false'
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1
+        with:
+          version: ${{ env.POETRY_TOOL_VERSION }}
+          virtualenvs-in-project: true
+      - name: Set up Python ${{ env.PYTHON_TOOL_VERSION }}
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
         with:
-          python-version: '3.8'
-      - run: sudo pip install black==23.1.0 ruff==0.0.287
-      - run: ruff check .
-      - run: black --check .
+          python-version: ${{ env.PYTHON_TOOL_VERSION }}
+          cache: poetry
+      - name: Cache Poetry dependencies
+        id: cache
+        uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
+        with:
+          path: .venv
+          key: setup-python-${{ runner.os }}-python-${{ env.PYTHON_TOOL_VERSION }}-poetry-v2-${{ hashFiles('**/poetry.lock') }}
+      - run: just backend-install
+      - run: just backend-lint
 
   backend-unit-tests:
     runs-on: ubuntu-22.04
     needs: backend-lint
-    env:
-      COMPOSE_FILE: .ci/compose.ci.yaml
-      COMPOSE_PROJECT_NAME: redash
-      COMPOSE_DOCKER_CLI_BUILD: 1
-      DOCKER_BUILDKIT: 1
+    permissions:
+      contents: read
     steps:
-      - if: github.event.pull_request.mergeable == 'false'
-        name: Exit if PR is not mergeable
-        run: exit 1
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: Build Docker Images
-        run: |
-          set -x
-          docker compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true
-          docker compose up -d
-          sleep 10
-      - name: Create Test Database
-        run: docker compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;"
-      - name: List Enabled Query Runners
-        run: docker compose -p redash run --rm redash manage ds list_types
-      - name: Run Tests
-        run: docker compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/
-      - name: Copy Test Results
-        run: |
-          mkdir -p /tmp/test-results/unit-tests
-          docker cp tests:/app/coverage.xml ./coverage.xml
-          docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
-      # - name: Upload coverage reports to Codecov
-      #   uses: codecov/codecov-action@v3
-      #   with:
-      #     token: ${{ secrets.CODECOV_TOKEN }}
-      - name: Store Test Results
-        uses: actions/upload-artifact@v4
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
-          name: backend-test-results
-          path: /tmp/test-results
-      - name: Store Coverage Results
-        uses: actions/upload-artifact@v4
+          persist-credentials: false
+      - name: Install Dependencies
+        uses: ./.github/composites/install
         with:
-          name: coverage
-          path: coverage.xml
+          install-yarn-deps: 'false'
+      - name: Run Backend Tests
+        run: just backend-test
 
   frontend-lint:
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
     steps:
-      - if: github.event.pull_request.mergeable == 'false'
-        name: Exit if PR is not mergeable
-        run: exit 1
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-          ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
-          node-version: ${{ env.NODE_VERSION }}
-          cache: 'yarn'
+          persist-credentials: false
       - name: Install Dependencies
-        run: |
-          npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
-      - name: Run Lint
-        run: yarn lint:ci
-      - name: Store Test Results
-        uses: actions/upload-artifact@v4
+        uses: ./.github/composites/install
         with:
-          name: frontend-test-results
-          path: /tmp/test-results
+          aws-role: arn:aws:iam::653993915282:role/ci-github-actions
+      - name: Run Frontend Lint
+        run: just frontend-lint
 
   frontend-unit-tests:
     runs-on: ubuntu-22.04
     needs: frontend-lint
+    permissions:
+      id-token: write
+      contents: read
     steps:
-      - if: github.event.pull_request.mergeable == 'false'
-        name: Exit if PR is not mergeable
-        run: exit 1
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-          ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
-          node-version: ${{ env.NODE_VERSION }}
-          cache: 'yarn'
+          persist-credentials: false
       - name: Install Dependencies
-        run: |
-          npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
-      - name: Run App Tests
-        run: yarn test
-      - name: Run Visualizations Tests
-        run: cd viz-lib && yarn test
-      - run: yarn lint
+        uses: ./.github/composites/install
+        with:
+          aws-role: arn:aws:iam::653993915282:role/ci-github-actions
+      - name: Run Frontend Tests
+        run: just frontend-test
 
   frontend-e2e-tests:
     runs-on: ubuntu-22.04
     needs: frontend-lint
-    env:
-      COMPOSE_FILE: .ci/compose.cypress.yaml
-      COMPOSE_PROJECT_NAME: cypress
-      CYPRESS_INSTALL_BINARY: 0
-      PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
-      # PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
-      # CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
-      # CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+    permissions:
+      id-token: write
+      contents: read
     steps:
-      - if: github.event.pull_request.mergeable == 'false'
-        name: Exit if PR is not mergeable
-        run: exit 1
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-          ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
-          node-version: ${{ env.NODE_VERSION }}
-          cache: 'yarn'
-      - name: Enable Code Coverage Report For Master Branch
-        if: endsWith(github.ref, '/master')
-        run: |
-          echo "CODE_COVERAGE=true" >> "$GITHUB_ENV"
+          persist-credentials: false
       - name: Install Dependencies
-        run: |
-          npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
-      - name: Setup Redash Server
-        run: |
-          set -x
-          yarn cypress build
-          yarn cypress start -- --skip-db-seed
-          docker compose run cypress yarn cypress db-seed
-      - name: Execute Cypress Tests
-        run: yarn cypress run-ci
-      - name: "Failure: output container logs to console"
-        if: failure()
-        run: docker compose logs
-      - name: Copy Code Coverage Results
-        run: docker cp cypress:/usr/src/app/coverage ./coverage || true
-      - name: Store Coverage Results
-        uses: actions/upload-artifact@v4
+        uses: ./.github/composites/install
         with:
-          name: coverage
-          path: coverage
+          aws-role: arn:aws:iam::653993915282:role/ci-github-actions
+      - name: Run Frontend E2E Tests
+        run: just frontend-e2e-test