diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 9ff9184782..710fee006e 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -8,16 +8,10 @@ on: required: true description: | The tag used to build the collector image - architectures: - type: string - required: true - description: | - Space-seperated list of architectures to build - outputs: collector-builder-tag: description: The builder tag used by the build - value: ${{ jobs.builder-needs-rebuilding.outputs.collector-builder-tag }} + value: ${{ jobs.build-builder-image.outputs.collector-builder-tag || 'master' }} env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -29,11 +23,7 @@ jobs: name: Determine if builder image needs to be built runs-on: ubuntu-24.04 outputs: - build-image: ${{ steps.builder-tag.outputs.build-image || false }} - collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag || 'master'}} - - env: - DEFAULT_BUILDER_TAG: master + build-image: ${{ steps.changed.outputs.builder-changed }} steps: - uses: actions/checkout@v4 @@ -48,46 +38,30 @@ jobs: - builder/Dockerfile - .github/workflows/collector-builder.yml - - name: Check labels and define builder tag - id: builder-tag - if: | - steps.changed.outputs.builder-changed == 'true' || - (github.event_name == 'push' && ( - github.ref_type == 'tag' || startsWith(github.ref_name, 'release-') - )) || - contains(github.event.pull_request.labels.*.name, 'build-builder-image') || - github.event_name == 'schedule' - run: | - COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" - if [[ "${{ github.event_name }}" == 'pull_request' || \ - "${{ github.ref_type }}" == 'tag' || \ - "${{ github.ref_name }}" =~ ^release- ]]; then - COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" - fi - - echo "Rebuild builder image with tag ${COLLECTOR_BUILDER_TAG}" - echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" - echo "build-image=true" >> "$GITHUB_OUTPUT" - build-builder-image: - name: Local builder image + name: Build the builder image + runs-on: ubuntu-24.04 # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: - builder-needs-rebuilding if: | - needs.builder-needs-rebuilding.outputs.build-image == 'true' + needs.builder-needs-rebuilding.outputs.build-image == 'true' || + (github.event_name == 'push' && ( + github.ref_type == 'tag' || startsWith(github.ref_name, 'release-') + )) || + contains(github.event.pull_request.labels.*.name, 'build-builder-image') || + github.event_name == 'schedule' + outputs: + collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} strategy: + fail-fast: false matrix: - arch: [amd64, arm64, ppc64le] - exclude: - - arch: ${{ contains(inputs.architectures, 'ppc64le') && '' || 'ppc64le' }} - runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} + arch: [amd64, ppc64le, s390x, arm64] env: PLATFORM: linux/${{ matrix.arch }} BUILD_TYPE: ci - COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} steps: - uses: actions/checkout@v4 @@ -102,54 +76,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Create ansible vars - run: | - cat << EOF > ${{ github.workspace }}/ansible/secrets.yml - --- - stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - collector_git_ref: ${{ github.ref }} - collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} - EOF - - - name: Build images - timeout-minutes: 480 - run: | - ansible-galaxy install -r ansible/requirements.yml - ansible-playbook \ - --connection local \ - -i localhost, \ - --limit localhost \ - -e arch='${{ matrix.arch }}' \ - -e @'${{ github.workspace }}/ansible/secrets.yml' \ - ansible/ci-build-builder.yml - - build-builder-image-remote-vm: - name: Remote builder image - # Multiarch builds sometimes take for eeeeeeeeeever - timeout-minutes: 480 - needs: - - builder-needs-rebuilding - if: | - needs.builder-needs-rebuilding.outputs.build-image == 'true' && - contains(inputs.architectures, 's390x') - strategy: - matrix: - arch: [s390x] - runs-on: ubuntu-24.04 - - env: - PLATFORM: linux/${{ matrix.arch }} - BUILD_TYPE: ci - COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/setup-python@v5 with: python-version: "3.10" @@ -175,22 +101,57 @@ jobs: job-tag: builder - name: Create Build VMs + if: | + matrix.arch == 's390x' && + (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) run: | make -C "${{ github.workspace }}/ansible" create-build-vms + - name: Define builder tag + id: builder-tag + run: | + COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" + if [[ "${{ github.event_name }}" == 'pull_request' || \ + "${{ github.ref_type }}" == 'tag' || \ + "${{ github.ref_name }}" =~ ^release- ]]; then + COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" + fi + + echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV" + echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" + - name: Create ansible vars run: | - cat << EOF > ${{ github.workspace }}/ansible/secrets.yml - --- - stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - collector_git_ref: ${{ github.ref }} - collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} - EOF + { + echo "---" + echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}" + echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}" + echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}" + echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}" + echo "collector_git_ref: ${{ github.ref }}" + echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}" + } > ${{ github.workspace }}/ansible/secrets.yml - name: Build images + if: | + (github.event_name != 'pull_request' && matrix.arch != 's390x') || + matrix.arch == 'amd64' || + (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x') + timeout-minutes: 480 + run: | + ansible-galaxy install -r ansible/requirements.yml + ansible-playbook \ + --connection local \ + -i localhost, \ + --limit localhost \ + -e arch='${{ matrix.arch }}' \ + -e @'${{ github.workspace }}/ansible/secrets.yml' \ + ansible/ci-build-builder.yml + + - name: Build s390x images + if: | + (github.event_name != 'pull_request' && matrix.arch == 's390x') || + (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x') timeout-minutes: 480 run: | ansible-playbook \ @@ -201,23 +162,22 @@ jobs: ansible/ci-build-builder.yml - name: Destroy VMs - if: always() + if: always() && matrix.arch == 's390x' run: | make -C ansible destroy-vms create-multiarch-manifest: needs: - - builder-needs-rebuilding - build-builder-image - - build-builder-image-remote-vm name: Create Multiarch manifest runs-on: ubuntu-24.04 if: | - always() && !contains(join(needs.*.result, ','), 'failure') && - needs.builder-needs-rebuilding.outputs.build-image == 'true' + github.event_name != 'pull_request' || + (needs.build-builder-image.outputs.collector-builder-tag != 'cache' && + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) env: - COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} - ARCHS: ${{ inputs.architectures }} + COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }} + ARCHS: amd64 ppc64le s390x arm64 steps: - uses: actions/checkout@v4 @@ -248,13 +208,45 @@ jobs: base-image: quay.io/rhacs-eng/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }} archs: ${{ env.ARCHS }} + retag-x86-image: + needs: + - build-builder-image + name: Retag x86 builder image + runs-on: ubuntu-24.04 + if: | + github.event_name == 'pull_request' && + needs.build-builder-image.outputs.collector-builder-tag != 'cache' && + !contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + env: + COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }} + steps: + - name: Pull image to retag + run: | + docker pull "quay.io/stackrox-io/collector-builder:${COLLECTOR_BUILDER_TAG}-amd64" + + - name: Retag and push stackrox-io + uses: stackrox/actions/images/retag-and-push@v1 + with: + src-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}-amd64 + dst-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }} + username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + + - name: Retag and push rhacs-eng + uses: stackrox/actions/images/retag-and-push@v1 + with: + src-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}-amd64 + dst-image: quay.io/rhacs-eng/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }} + username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + notify: runs-on: ubuntu-24.04 if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-builder-image - - build-builder-image-remote-vm - create-multiarch-manifest + - retag-x86-image steps: - name: Slack notification uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 736f68d1cb..0a401ca01b 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -18,11 +18,6 @@ on: required: true description: | The builder tag to use in the build - architectures: - type: string - required: true - description: | - Space-seperated list of architectures to build env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -34,13 +29,12 @@ env: jobs: build-collector-image: - name: Local collector image ${{ matrix.arch }} + name: Build Collector + runs-on: ubuntu-24.04 strategy: + fail-fast: false matrix: - arch: [amd64, arm64, ppc64le] - exclude: - - arch: ${{ contains(inputs.architectures, 'ppc64le') && '' || 'ppc64le' }} - runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} + arch: [amd64, ppc64le, arm64] env: PLATFORM: linux/${{ matrix.arch }} @@ -68,7 +62,6 @@ jobs: rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} collector_git_ref: ${{ github.ref }} - collector_git_sha: ${{ github.sha }} collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }} rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} @@ -78,7 +71,11 @@ jobs: driver_version: ${DRIVER_VERSION} EOF - - name: Build ${{ matrix.arch }} image locally + - name: Build images + if: | + github.event_name != 'pull_request' || + matrix.arch == 'amd64' || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') timeout-minutes: 480 run: | ansible-playbook \ @@ -90,10 +87,11 @@ jobs: ansible/ci-build-collector.yml build-collector-image-remote-vm: - name: Remote collector image + name: Build Collector on a remote VM runs-on: ubuntu-24.04 - if: contains(inputs.architectures, 's390x') + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') strategy: + fail-fast: false matrix: arch: [s390x] @@ -170,9 +168,11 @@ jobs: - build-collector-image-remote-vm name: Create Multiarch manifest runs-on: ubuntu-24.04 - if: always() && !contains(join(needs.*.result, ','), 'failure') + if: | + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') env: - ARCHS: ${{ inputs.architectures }} + ARCHS: amd64 ppc64le s390x arm64 steps: - uses: actions/checkout@v4 @@ -203,6 +203,35 @@ jobs: base-image: quay.io/rhacs-eng/collector:${{ inputs.collector-tag }} archs: ${{ env.ARCHS }} + retag-x86-image: + needs: + - build-collector-image + name: Retag x86 image + runs-on: ubuntu-24.04 + if: | + github.event_name == 'pull_request' && + !contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + steps: + - name: Pull image to retag + run: | + docker pull ${{ inputs.collector-image }}-amd64 + + - name: Retag and push stackrox-io + uses: stackrox/actions/images/retag-and-push@v1 + with: + src-image: ${{ inputs.collector-image }}-amd64 + dst-image: ${{ inputs.collector-image }} + username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + + - name: Retag and push rhacs-eng + uses: stackrox/actions/images/retag-and-push@v1 + with: + src-image: ${{ inputs.collector-image }}-amd64 + dst-image: ${{ env.RHACS_ENG_IMAGE }} + username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + notify: runs-on: ubuntu-24.04 if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' @@ -210,6 +239,7 @@ jobs: - build-collector-image - build-collector-image-remote-vm - create-multiarch-manifest + - retag-x86-image steps: - name: Slack notification uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index b9bea60ec6..be09716158 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -58,13 +58,6 @@ on: description: | Trigger rebuild of QA containers value: ${{ jobs.common-variables.outputs.rebuild-qa-containers }} - architectures: - description: | - Space-seperated list of architectures to build - value: | - ${{ ((github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && - 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} jobs: common-variables: diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a8f17c96b9..1290e8e7de 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -91,6 +91,9 @@ jobs: arm64-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml + if: | + github.event_name != 'pull_request' || inputs.is-konflux || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') strategy: # ensure that if one part of the matrix fails, the # rest will continue diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9d8e3384d..f002e6eb7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,6 @@ jobs: needs: init with: collector-tag: ${{ needs.init.outputs.collector-tag }} - architectures: ${{ needs.init.outputs.architectures }} secrets: inherit build-collector: @@ -52,7 +51,6 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-image: ${{ needs.init.outputs.collector-image }} collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - architectures: ${{ needs.init.outputs.architectures }} secrets: inherit build-test-containers: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d13562ff21..5ac589df89 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -11,18 +11,16 @@ on: jobs: unit-tests: + runs-on: ubuntu-24.04 container: image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} strategy: fail-fast: false matrix: - arch: [amd64, arm64] cmake-flags: - -DCMAKE_BUILD_TYPE=Release - -DADDRESS_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug - -DUSE_VALGRIND=ON -DCMAKE_BUILD_TYPE=Debug - runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} - steps: - uses: actions/checkout@v4 with: