diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 0bc24fc36c..ae1aa487c8 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -8,11 +8,6 @@ on: required: true description: | The tag used to build the collector image - architectures: - type: string - required: true - description: | - Space-separated list of architectures to build outputs: collector-builder-tag: @@ -31,7 +26,6 @@ jobs: outputs: build-image: ${{ steps.builder-tag.outputs.build-image || false }} collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag || 'master'}} - local-exclude: ${{ steps.arch.outputs.local-exclude || '[]'}} env: DEFAULT_BUILDER_TAG: master @@ -70,24 +64,23 @@ jobs: echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" echo "build-image=true" >> "$GITHUB_OUTPUT" - - name: Check arches for local build - if: ${{ ! contains(inputs.architectures, 'ppc64le') }} - id: arch - run: echo 'local-exclude=[{"arch":"ppc64le"}]' >> "$GITHUB_OUTPUT" - build-builder-image: - name: Local builder image - # Multiarch builds sometimes take for eeeeeeeeeever - timeout-minutes: 480 + name: Build builder image needs: - builder-needs-rebuilding if: | needs.builder-needs-rebuilding.outputs.build-image == 'true' strategy: matrix: - arch: [amd64, arm64, ppc64le] - exclude: ${{ fromJSON(needs.builder-needs-rebuilding.outputs.local-exclude) }} - runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} + arch: + - amd64 + - arm64 + - ppc64le + - s390x + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || + (matrix.arch == 'ppc64le' && 'ubuntu-24.04-ppc64le') || + (matrix.arch == 's390x' && 'ubuntu-24.04-s390x') || + 'ubuntu-24.04' }} env: PLATFORM: linux/${{ matrix.arch }} @@ -99,14 +92,14 @@ jobs: with: submodules: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:qemu-v8.1.5 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Install ansible on P&Z runners + if: matrix.arch == 'ppc64le' || matrix.arch == 's390x' + run: | + sudo apt-get install -y ansible + - name: Create ansible vars run: | cat << EOF > ${{ github.workspace }}/ansible/secrets.yml @@ -120,7 +113,6 @@ jobs: EOF - name: Build images - timeout-minutes: 480 run: | ansible-galaxy install -r ansible/requirements.yml ansible-playbook \ @@ -131,90 +123,10 @@ jobs: -e @'${{ github.workspace }}/ansible/secrets.yml' \ ansible/ci-build-builder.yml - build-builder-image-remote-vm: - name: Remote builder image - # Multiarch builds sometimes take for eeeeeeeeeever - timeout-minutes: 480 - needs: - - builder-needs-rebuilding - if: | - needs.builder-needs-rebuilding.outputs.build-image == 'true' && - contains(inputs.architectures, 's390x') - strategy: - matrix: - arch: [s390x] - runs-on: ubuntu-24.04 - - env: - PLATFORM: linux/${{ matrix.arch }} - BUILD_TYPE: ci - COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - uses: 'google-github-actions/auth@v2' - with: - credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' - - - uses: 'google-github-actions/setup-gcloud@v2' - - - uses: ./.github/actions/setup-vm-creds - with: - gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }} - gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }} - s390x-ssh-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }} - ppc64le-ssh-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }} - ppc64le-ssh-key-pub: ${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }} - s390x-key: ${{ secrets.IBM_CLOUD_S390x_API_KEY }} - ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }} - redhat-username: ${{ secrets.REDHAT_USERNAME }} - redhat-password: ${{ secrets.REDHAT_PASSWORD }} - vm-type: all - job-tag: builder - - - name: Create Build VMs - run: | - make -C "${{ github.workspace }}/ansible" create-build-vms - - - name: Create ansible vars - run: | - cat << EOF > ${{ github.workspace }}/ansible/secrets.yml - --- - stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - collector_git_ref: ${{ github.ref }} - collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} - EOF - - - name: Build images - timeout-minutes: 480 - run: | - ansible-playbook \ - -i ansible/ci \ - -e build_hosts='job_id_${{ env.JOB_ID }}' \ - -e arch='${{ matrix.arch }}' \ - -e @'${{ github.workspace }}/ansible/secrets.yml' \ - ansible/ci-build-builder.yml - - - name: Destroy VMs - if: always() - run: | - make -C ansible destroy-vms - create-multiarch-manifest: needs: - builder-needs-rebuilding - build-builder-image - - build-builder-image-remote-vm name: Create Multiarch manifest runs-on: ubuntu-24.04 if: | @@ -222,7 +134,7 @@ jobs: needs.builder-needs-rebuilding.outputs.build-image == 'true' env: COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} - ARCHS: ${{ inputs.architectures }} + ARCHS: amd64 arm64 ppc64le s390x steps: - uses: actions/checkout@v4 @@ -258,7 +170,6 @@ jobs: if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-builder-image - - build-builder-image-remote-vm - create-multiarch-manifest steps: - name: Slack notification diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index b97590eac3..a1ee87c037 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -18,11 +18,6 @@ on: required: true description: | The builder tag to use in the build - architectures: - type: string - required: true - description: | - Space-separated list of architectures to build env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -33,26 +28,19 @@ env: ADDRESS_SANITIZER: ${{ contains(github.event.pull_request.labels.*.name, 'address-sanitizer') }} jobs: - prepare-build-collector: - name: Prepare builders for collector - runs-on: ubuntu-24.04 - outputs: - local-exclude: ${{ steps.arch.outputs.local-exclude || '[]' }} - - steps: - - name: Check arches for local build - if: ${{ ! contains(inputs.architectures, 'ppc64le') }} - id: arch - run: echo 'local-exclude=[{"arch":"ppc64le"}]' >> "$GITHUB_OUTPUT" - build-collector-image: - name: Local collector image ${{ matrix.arch }} - needs: prepare-build-collector + name: Build collector image ${{ matrix.arch }} strategy: matrix: - arch: [amd64, arm64, ppc64le] - exclude: ${{ fromJSON(needs.prepare-build-collector.outputs.local-exclude) }} - runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} + arch: + - amd64 + - arm64 + - ppc64le + - s390x + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || + (matrix.arch == 'ppc64le' && 'ubuntu-24.04-ppc64le') || + (matrix.arch == 's390x' && 'ubuntu-24.04-s390x') || + 'ubuntu-24.04' }} env: PLATFORM: linux/${{ matrix.arch }} @@ -63,14 +51,14 @@ jobs: with: submodules: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:qemu-v8.1.5 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Install ansible on P&Z runners + if: matrix.arch == 'ppc64le' || matrix.arch == 's390x' + run: | + sudo apt-get install -y ansible + - name: Create ansible vars run: | cat << EOF > ${{ github.workspace }}/ansible/secrets.yml @@ -91,7 +79,6 @@ jobs: EOF - name: Build ${{ matrix.arch }} image locally - timeout-minutes: 480 run: | ansible-playbook \ --connection local \ @@ -101,90 +88,14 @@ jobs: -e @'${{ github.workspace }}/ansible/secrets.yml' \ ansible/ci-build-collector.yml - build-collector-image-remote-vm: - name: Remote collector image - runs-on: ubuntu-24.04 - if: contains(inputs.architectures, 's390x') - strategy: - matrix: - arch: [s390x] - - env: - PLATFORM: linux/${{ matrix.arch }} - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - uses: 'google-github-actions/auth@v2' - with: - credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' - - - uses: 'google-github-actions/setup-gcloud@v2' - - - uses: ./.github/actions/setup-vm-creds - with: - gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }} - gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }} - s390x-ssh-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }} - ppc64le-ssh-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }} - ppc64le-ssh-key-pub: ${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }} - s390x-key: ${{ secrets.IBM_CLOUD_S390x_API_KEY }} - ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }} - redhat-username: ${{ secrets.REDHAT_USERNAME }} - redhat-password: ${{ secrets.REDHAT_PASSWORD }} - vm-type: rhel-${{ matrix.arch }} - job-tag: builder - - - name: Create Build VMs - run: | - make -C "${{ github.workspace }}/ansible" create-build-vms - - - name: Create ansible vars - run: | - cat << EOF > ${{ github.workspace }}/ansible/secrets.yml - --- - stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - collector_git_ref: ${{ github.ref }} - collector_git_sha: ${{ github.sha }} - collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} - disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }} - rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} - collector_image: ${{ inputs.collector-image }} - collector_tag: ${{ inputs.collector-tag }} - debug_mode: ${{ github.event_name == 'pull_request' }} - EOF - - - name: Build ${{ matrix.arch }} image - timeout-minutes: 480 - run: | - ansible-playbook \ - -i ansible/ci \ - -e arch='${{ matrix.arch }}' \ - -e build_hosts='job_id_${{ env.JOB_ID }}' \ - -e @'${{ github.workspace }}/ansible/secrets.yml' \ - ansible/ci-build-collector.yml - - - name: Destroy Build VMs - if: always() - run: | - make -C ansible destroy-vms - create-multiarch-manifest: needs: - build-collector-image - - build-collector-image-remote-vm name: Create Multiarch manifest runs-on: ubuntu-24.04 if: always() && !contains(join(needs.*.result, ','), 'failure') env: - ARCHS: ${{ inputs.architectures }} + ARCHS: amd64 arm64 ppc64le s390x steps: - uses: actions/checkout@v4 @@ -220,7 +131,6 @@ jobs: if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-collector-image - - build-collector-image-remote-vm - create-multiarch-manifest steps: - name: Slack notification diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index 1db7bcd88b..be09716158 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -58,13 +58,6 @@ on: description: | Trigger rebuild of QA containers value: ${{ jobs.common-variables.outputs.rebuild-qa-containers }} - architectures: - description: | - Space-separated list of architectures to build - value: | - ${{ ((github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && - 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} jobs: common-variables: diff --git a/.github/workflows/integration-test-containers.yml b/.github/workflows/integration-test-containers.yml index bda7c2b62b..9d6a1590fc 100644 --- a/.github/workflows/integration-test-containers.yml +++ b/.github/workflows/integration-test-containers.yml @@ -80,6 +80,13 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@v3 + - name: Setup go and its cache + uses: actions/setup-go@v5 + with: + go-version-file: integration-tests/go.mod + cache-dependency-path: | + integration-tests/go.sum + - name: Create Ansible Vars (inc. Secrets) run: | cat << EOF > ${{ github.workspace }}/ansible/secrets.yml @@ -105,28 +112,6 @@ jobs: echo "COLLECTOR_TESTS_TAG=${COLLECTOR_TESTS_TAG}" >> "$GITHUB_ENV" echo "collector-tests-tag=${COLLECTOR_TESTS_TAG}" >> "$GITHUB_OUTPUT" - - name: Check if multiarch is needed - run: | - BUILD_MULTI_ARCH="false" - - if [[ "${GITHUB_EVENT_NAME}" != "pull_request" ]]; then - BUILD_MULTI_ARCH="true" - fi - - if [[ "${{ inputs.is-konflux }}" == "true" ]]; then - BUILD_MULTI_ARCH="true" - fi - - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') }}" == "true" ]]; then - BUILD_MULTI_ARCH="true" - fi - - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'run-cpaas-steps') }}" == "true" ]]; then - BUILD_MULTI_ARCH="true" - fi - - echo "BUILD_MULTI_ARCH=${BUILD_MULTI_ARCH}" >> "$GITHUB_ENV" - - name: Build images run: | ansible-galaxy install -r ansible/requirements.yml @@ -135,7 +120,6 @@ jobs: ansible-playbook \ --connection local -i localhost, --limit localhost \ -e test_image="quay.io/rhacs-eng/collector-tests:${COLLECTOR_TESTS_TAG}" \ - -e "{\"build_multi_arch\": $BUILD_MULTI_ARCH}" \ -e @'${{ github.workspace }}/ansible/secrets.yml' \ ansible/ci-build-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8c1c90d255..3af1f8bd55 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -113,9 +113,6 @@ jobs: s390x-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: | - github.event_name != 'pull_request' || inputs.is-konflux || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') with: vm_type: rhel-s390x collector-tag: ${{ inputs.collector-tag }} @@ -127,9 +124,6 @@ jobs: ppc64le-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: | - github.event_name != 'pull_request' || inputs.is-konflux || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') with: vm_type: rhel-ppc64le collector-tag: ${{ inputs.collector-tag }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9d8e3384d..f002e6eb7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,6 @@ jobs: needs: init with: collector-tag: ${{ needs.init.outputs.collector-tag }} - architectures: ${{ needs.init.outputs.architectures }} secrets: inherit build-collector: @@ -52,7 +51,6 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-image: ${{ needs.init.outputs.collector-image }} collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - architectures: ${{ needs.init.outputs.architectures }} secrets: inherit build-test-containers: diff --git a/ansible/ci-build-builder.yml b/ansible/ci-build-builder.yml index d91284aaee..28706cea0d 100644 --- a/ansible/ci-build-builder.yml +++ b/ansible/ci-build-builder.yml @@ -12,18 +12,6 @@ local_branch: local tasks: - - name: Clone repository - ansible.builtin.git: - repo: https://github.com/stackrox/collector - dest: "{{ collector_root }}" - # We fetch the ref (either master, or pull//merge) and then - # create a local branch based on that. Doing it this way, rather - # than with commit hashes, prevents "reference is not a tree" errors - version: "{{ local_branch }}" - refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}" - recursive: true - when: arch == "s390x" - - name: Build the collector builder image community.general.make: chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}" diff --git a/ansible/ci-build-collector.yml b/ansible/ci-build-collector.yml index 631bbffed0..efe43cd54c 100644 --- a/ansible/ci-build-collector.yml +++ b/ansible/ci-build-collector.yml @@ -14,19 +14,6 @@ local_branch: local tasks: - - debug: var=collector_root - - name: Clone repository - ansible.builtin.git: - repo: https://github.com/stackrox/collector - dest: "{{ collector_root }}" - # We fetch the ref (either master, or pull//merge) and then - # create a local branch based on that. Doing it this way, rather - # than with commit hashes, prevents "reference is not a tree" errors - version: "{{ local_branch }}" - refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}" - recursive: true - when: arch == "s390x" - - name: Run the builder image community.general.make: chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}" diff --git a/ansible/ci-build-tests.yml b/ansible/ci-build-tests.yml index a61720aa4b..4d5cdf607a 100644 --- a/ansible/ci-build-tests.yml +++ b/ansible/ci-build-tests.yml @@ -5,17 +5,15 @@ environment: tasks: - - set_fact: - platforms: "linux/amd64,linux/ppc64le,linux/s390x,linux/arm64" - when: build_multi_arch - - - set_fact: - platforms: "linux/amd64,linux/arm64" - when: not build_multi_arch - - set_fact: collector_root: "{{ lookup('env', 'GITHUB_WORKSPACE') }}" + - name: Build test binaries + community.general.make: + chdir: "{{ collector_root }}/integration-tests" + target: build-all + jobs: "{{ ansible_facts['processor_vcpus'] }}" + - name: Login to quay.io community.docker.docker_login: registry_url: quay.io @@ -31,7 +29,7 @@ # - we can push the images and manifest in a single command. The make # target can be used for local development and testing. docker buildx build --push \ - --platform "{{ platforms }}" \ + --platform "linux/amd64,linux/arm64,linux/ppc64le,linux/s390x" \ -t '{{ test_image }}' \ {{ collector_root }}/integration-tests register: build_result @@ -45,4 +43,3 @@ registry_url: quay.io state: absent when: true - diff --git a/integration-tests/Dockerfile b/integration-tests/Dockerfile index ca3845a9f5..5a1bc0769c 100644 --- a/integration-tests/Dockerfile +++ b/integration-tests/Dockerfile @@ -1,35 +1,15 @@ -ARG TEST_ROOT="/tests" - -FROM golang:1.23 as builder +FROM alpine:3.18 -ARG TEST_ROOT +ARG TARGETARCH +ARG TEST_ROOT="/tests" -ENV GOCACHE=/root/.cache/go-build +RUN apk add docker && \ + mkdir -p $TEST_ROOT -RUN mkdir -p "$TEST_ROOT" WORKDIR "$TEST_ROOT" -# Cache dependencies -COPY go.* "$TEST_ROOT" -RUN go mod download - -COPY suites "$TEST_ROOT/suites/" -COPY pkg "$TEST_ROOT/pkg/" -COPY integration_test.go "$TEST_ROOT" -COPY benchmark_test.go "$TEST_ROOT" -COPY k8s_test.go "$TEST_ROOT" - -RUN --mount=type=cache,target="/root/.cache/go-build" CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go test -tags bench,k8s -c -o collector-tests +COPY bin/$TARGETARCH/collector-tests /usr/local/bin -FROM alpine:3.18 - -ARG TEST_ROOT - -RUN apk add docker - -COPY --from=builder $TEST_ROOT/collector-tests $TEST_ROOT/collector-tests -COPY images.yml "$TEST_ROOT" - -WORKDIR "$TEST_ROOT" +COPY images.yml . -ENTRYPOINT ["./collector-tests"] +ENTRYPOINT ["collector-tests"] diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 2ea178de94..6a8e992bce 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -21,6 +21,8 @@ ifeq ($(COLLECTOR_TESTS_IMAGE),) COLLECTOR_TESTS_IMAGE=$(COLLECTOR_TESTS_REPO):$(COLLECTOR_TAG) endif +GOARCH ?= $(shell go env GOARCH) + SHELL=/bin/bash # Environment variable COLLECTOR_IMAGE is used by integration-tests @@ -59,11 +61,21 @@ $(foreach element,$(ALL_TESTS),$(eval $(call make-test-target-dockerized,$(eleme .PHONY: build build: - mkdir -p bin - go test -tags bench,k8s -c -o bin/collector-tests + mkdir -p bin/$(GOARCH) + CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go test -tags bench,k8s -c -o bin/$(GOARCH)/collector-tests + +ARCHS := amd64 arm64 ppc64le s390x + +build-%: + GOARCH=$(*) make build + + +build-all: $(addprefix build-,$(ARCHS)) + +.PHONY: build-all build-% .PHONY: build-image -build-image: +build-image: build-all docker build --platform $(PLATFORM) -t $(COLLECTOR_TESTS_IMAGE) \ --build-arg QA_TAG=$(shell cat container/QA_TAG) \ $(CURDIR)