diff --git a/.github/config/hive/clients.yaml b/.github/config/hive/clients.yaml new file mode 100644 index 00000000..8d25fc55 --- /dev/null +++ b/.github/config/hive/clients.yaml @@ -0,0 +1,4 @@ +- client: ethlambda + nametag: devnet3 +- client: ethlambda + nametag: devnet4 diff --git a/.github/scripts/check-hive-results.sh b/.github/scripts/check-hive-results.sh new file mode 100644 index 00000000..d1cb3e21 --- /dev/null +++ b/.github/scripts/check-hive-results.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +set -euo pipefail + +results_dir="${1:-src/results}" + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to parse Hive results but was not found in PATH" + exit 1 +fi + +if [ ! -d "${results_dir}" ]; then + echo "Hive results directory '${results_dir}' not found" + exit 1 +fi + +shopt -s nullglob +json_files=("${results_dir}"/*.json) +shopt -u nullglob + +if [ "${#json_files[@]}" -eq 0 ]; then + echo "No Hive JSON result files found in ${results_dir}" + exit 1 +fi + +failures=0 +failed_logs_root="${results_dir}/failed_logs" +rm -rf "${failed_logs_root}" +mkdir -p "${failed_logs_root}" + +for json_file in "${json_files[@]}"; do + if [[ "${json_file}" == *"hive.json" ]]; then + continue + fi + + suite_name="$(jq -r '.name // empty' "${json_file}")" + suite_label="${suite_name:-$(basename "${json_file}" .json)}" + failed_cases="$(jq '[.testCases[]? | select(.summaryResult.pass != true)] | length' "${json_file}")" + + if [ "${failed_cases}" -eq 0 ]; then + continue + fi + + failures=$((failures + failed_cases)) + echo "Detected ${failed_cases} failing Hive test case(s) in ${suite_label}" + + failure_list="$( + jq -r ' + .testCases[]? + | select(.summaryResult.pass != true) + | . as $case + | ($case.summaryResult // {}) as $summary + | ($summary.message // $summary.reason // $summary.error // "") as $message + | "- " + ($case.name // "unknown test") + + (if $message != "" then ": " + $message else "" end) + ' "${json_file}" + )" + + printf '%s\n' "${failure_list}" + + suite_slug="$(printf '%s' "${suite_label}" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9._-]+/-/g; s/^-//; s/-$//')" + suite_dir="${failed_logs_root}/${suite_slug:-suite}" + mkdir -p "${suite_dir}" + + cp "${json_file}" "${suite_dir}/" + printf '%s\n\n%s\n' "Detected ${failed_cases} failing Hive test case(s) in ${suite_label}" "${failure_list}" > "${suite_dir}/failed-tests.txt" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Hive failures: ${suite_label}" + printf '%s\n' "${failure_list}" + echo + } >> "${GITHUB_STEP_SUMMARY}" + fi +done + +if [ "${failures}" -gt 0 ]; then + echo "Hive reported ${failures} failing test case(s) in total" + exit 1 +fi + +echo "Hive reported no failing test cases." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0c70608..4d8a3117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,3 +146,128 @@ jobs: - name: Run tests run: make test + + docker_build: + name: Build Docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build ethlambda Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: | + GIT_COMMIT=${{ github.sha }} + GIT_BRANCH=${{ github.ref_name }} + push: false + tags: | + ghcr.io/lambdaclass/ethlambda:devnet3 + ghcr.io/lambdaclass/ethlambda:devnet4 + outputs: type=docker,dest=/tmp/ethlambda_image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload image artifacts + uses: actions/upload-artifact@v6 + with: + name: ethlambda_image + path: /tmp/ethlambda_image.tar + + run-hive: + name: Hive - ${{ matrix.name }} + runs-on: ubuntu-latest + needs: docker_build + strategy: + fail-fast: false + matrix: + include: + - name: "Lean RPC Compat" + limit: "rpc-compat" + artifact_prefix: lean_rpc_compat + - name: "Lean Sync" + limit: "sync" + artifact_prefix: lean_sync + - name: "Lean Client Interop" + limit: "client-interop" + artifact_prefix: lean_client_interop + - name: "Lean Validation" + limit: "validation" + artifact_prefix: lean_validation + - name: "Lean Gossip" + limit: "gossip" + artifact_prefix: lean_gossip + - name: "Lean ReqResp" + limit: "reqresp" + artifact_prefix: lean_reqresp + - name: "Lean Fork Choice Spec Tests" + limit: "lean-spec-tests-fork-choice" + artifact_prefix: lean_spec_fork_choice + - name: "Lean State Transition Spec Tests" + limit: "lean-spec-tests-state-transition" + artifact_prefix: lean_spec_state_transition + - name: "Lean Verify Signatures Spec Tests" + limit: "lean-spec-tests-verify-signatures" + artifact_prefix: lean_spec_verify_signatures + steps: + - uses: actions/checkout@v6 + + - name: Download ethlambda image artifact + uses: actions/download-artifact@v6 + with: + name: ethlambda_image + path: /tmp + + - name: Load image + run: docker load --input /tmp/ethlambda_image.tar + + - name: Load hive client config + id: client-config + shell: bash + run: | + { + echo "config<> "$GITHUB_OUTPUT" + + - name: Determine hive flags + id: hive-flags + shell: bash + env: + SIM_LIMIT: ${{ matrix.limit }} + run: | + FLAGS="--sim.parallelism 4 --sim.loglevel 3" + if [[ -n "$SIM_LIMIT" ]]; then + escaped_limit=${SIM_LIMIT//\'/\'\\\'\'} + FLAGS+=" --sim.limit '$escaped_limit'" + fi + echo "flags=$FLAGS" >> "$GITHUB_OUTPUT" + + - name: Run Hive Simulation + uses: ethpandaops/hive-github-action@v0.5.0 + with: + hive_repository: ethereum/hive + hive_version: 6f704d0cd8fb4dfd3f635abcc5a3ea6cd395e61b + simulator: lean + client: ethlambda + client_config: ${{ steps.client-config.outputs.config }} + extra_flags: ${{ steps.hive-flags.outputs.flags }} + + - name: Check Hive Results For Failures + id: verify-hive-results + if: ${{ success() }} + shell: bash + run: bash ./.github/scripts/check-hive-results.sh src/results + + - name: Upload Hive Failure Logs + if: ${{ failure() && steps.verify-hive-results.conclusion == 'failure' }} + uses: actions/upload-artifact@v6 + with: + name: hive_failed_logs_${{ matrix.artifact_prefix }} + path: src/results/failed_logs + if-no-files-found: warn