From cd7a867fe6821c1d75fcb1eaf11fb5b0b4ff22e3 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 13 Feb 2025 13:00:34 -0800 Subject: [PATCH 1/6] replace fetcher --- .github/workflows/android-perf.yml | 123 ++++++++++++++++------------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 201fb3b7a8f..083c8e621cf 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -96,63 +96,6 @@ jobs: PYTHONPATH="${PWD}" python .ci/scripts/gather_benchmark_configs.py $ARGS - prepare-test-specs: - runs-on: linux.2xlarge - needs: set-parameters - strategy: - matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} - fail-fast: false - steps: - - uses: actions/checkout@v3 - - - name: Prepare the spec - id: prepare - shell: bash - env: - BENCHMARK_CONFIG: ${{ toJSON(matrix) }} - working-directory: extension/benchmark/android/benchmark - run: | - set -eux - - # The model will be exported in the next step to this S3 path - MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" - # We could write a script to properly use jinja here, but there is only one variable, - # so let's just sed it - sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2 - - BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') - # The config for this benchmark runs, we save it in the test spec so that it can be fetched - # later by the upload script - sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2 - - cp android-llm-device-farm-test-spec.yml.j2 android-llm-device-farm-test-spec.yml - # Just print the test spec for debugging - cat android-llm-device-farm-test-spec.yml - - # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" - echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT - - - name: Upload the spec - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml - - - name: Update the benchmark configs - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json - export-models: name: export-models uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main @@ -335,6 +278,72 @@ jobs: fi echo "::endgroup::" + prepare-test-specs: + runs-on: linux.2xlarge + needs: + - set-parameters + - export-models + strategy: + matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Prepare the spec + id: prepare + shell: bash + env: + BENCHMARK_CONFIG: ${{ toJSON(matrix) }} + working-directory: extension/benchmark/android/benchmark + run: | + set -eux + + # The model will be exported in the next step to this S3 path + MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" + + # Check if the model artifact is successfully uploaded from previous step, failed to create the test-spec if it does not exist. + if ! curl -s --head -f ${MODEL_PATH}; then + echo "failed to retrieve ${MODEL_PATH}" + exit 1; + fi + + # We could write a script to properly use jinja here, but there is only one variable, + # so let's just sed it + sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2 + + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') + # The config for this benchmark runs, we save it in the test spec so that it can be fetched + # later by the upload script + sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2 + + cp android-llm-device-farm-test-spec.yml.j2 android-llm-device-farm-test-spec.yml + # Just print the test spec for debugging + cat android-llm-device-farm-test-spec.yml + + # Save the benchmark configs so that we can use it later in the dashboard + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT + + - name: Upload the spec + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml + + - name: Update the benchmark configs + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json + build-benchmark-app: name: build-benchmark-app uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main From 43871791d9411ca87e766f0f1b66801181c34f95 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 13 Feb 2025 13:32:18 -0800 Subject: [PATCH 2/6] replace fetcher --- .github/workflows/android-perf.yml | 2 +- .../benchmark/android-llm-device-farm-test-spec.yml.j2 | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 083c8e621cf..c292712db01 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -301,7 +301,7 @@ jobs: # The model will be exported in the next step to this S3 path MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" - # Check if the model artifact is successfully uploaded from previous step, failed to create the test-spec if it does not exist. + # Check if the model artifact is successfully uploaded from previous step, failed to upload the test-spec if it does not exist. if ! curl -s --head -f ${MODEL_PATH}; then echo "failed to retrieve ${MODEL_PATH}" exit 1; diff --git a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 index ae25a071e5c..11d2acbcccc 100644 --- a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 +++ b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 @@ -11,7 +11,12 @@ phases: # Print this so that the upload script can read and process the benchmark config - echo "The benchmark config is {{ benchmark_config_id }}" - # Download the model from S3 + - echo "Checking if model exists in S3..." + - | + if ! curl -s --fail -I '{{ model_path }}'; then + echo "model file not found in S3. Exiting test." + exit 1 + fi - curl -s --fail '{{ model_path }}' -o model.zip - unzip model.zip && ls -la From b5777f7c150dae1a60ad0550440b587abc351f0c Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 13 Feb 2025 13:35:27 -0800 Subject: [PATCH 3/6] replace fetcher --- .../android/benchmark/android-llm-device-farm-test-spec.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 index 11d2acbcccc..a45e6d3f711 100644 --- a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 +++ b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 @@ -17,6 +17,7 @@ phases: echo "model file not found in S3. Exiting test." exit 1 fi + # Download the model from S3 - curl -s --fail '{{ model_path }}' -o model.zip - unzip model.zip && ls -la From 0a568356721015b486d3636052fe05c21c8af737 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 13 Feb 2025 14:14:18 -0800 Subject: [PATCH 4/6] replace fetcher --- .../benchmark/android-llm-device-farm-test-spec.yml.j2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 index a45e6d3f711..ae25a071e5c 100644 --- a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 +++ b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 @@ -11,12 +11,6 @@ phases: # Print this so that the upload script can read and process the benchmark config - echo "The benchmark config is {{ benchmark_config_id }}" - - echo "Checking if model exists in S3..." - - | - if ! curl -s --fail -I '{{ model_path }}'; then - echo "model file not found in S3. Exiting test." - exit 1 - fi # Download the model from S3 - curl -s --fail '{{ model_path }}' -o model.zip - unzip model.zip && ls -la From ee849e68784aad9de396d5da2e28e18dd701c89f Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 13 Feb 2025 14:51:36 -0800 Subject: [PATCH 5/6] replace fetcher --- .github/workflows/android-perf.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index c292712db01..d13b799b5ca 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -301,11 +301,8 @@ jobs: # The model will be exported in the next step to this S3 path MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" - # Check if the model artifact is successfully uploaded from previous step, failed to upload the test-spec if it does not exist. - if ! curl -s --head -f ${MODEL_PATH}; then - echo "failed to retrieve ${MODEL_PATH}" - exit 1; - fi + # Check if the model artifact exists, fail this step skip generating test-spec. + curl -s --head -f ${MODEL_PATH} # We could write a script to properly use jinja here, but there is only one variable, # so let's just sed it From f2b250e4a5d3a6028100491afcb2de2ada343d9d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 13 Feb 2025 15:21:36 -0800 Subject: [PATCH 6/6] replace fetcher --- .github/workflows/apple-perf.yml | 119 ++++++++++++++++--------------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index ea88be441cb..e0abea115e6 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -98,63 +98,6 @@ jobs: echo "benchmark_configs is: ${{ steps.set-parameters.outputs.benchmark_configs }}" - prepare-test-specs: - runs-on: linux.2xlarge - needs: set-parameters - strategy: - matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} - fail-fast: false - steps: - - uses: actions/checkout@v3 - - - name: Prepare the spec - id: prepare - shell: bash - env: - BENCHMARK_CONFIG: ${{ toJSON(matrix) }} - working-directory: extension/benchmark/apple/Benchmark - run: | - set -eux - - # The model will be exported in the next step to this S3 path - MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" - # We could write a script to properly use jinja here, but there is only one variable, - # so let's just sed it - sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2 - - BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') - # The config for this benchmark runs, we save it in the test spec so that it can be fetched - # later by the upload script - sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2 - - cp default-ios-device-farm-appium-test-spec.yml.j2 default-ios-device-farm-appium-test-spec.yml - # Just print the test spec for debugging - cat default-ios-device-farm-appium-test-spec.yml - - # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" - echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT - - - name: Upload the spec - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml - - - name: Update the benchmark configs - uses: seemethere/upload-artifact-s3@v5 - with: - s3-bucket: gha-artifacts - s3-prefix: | - ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ - retention-days: 1 - if-no-files-found: error - path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json - export-models: name: export-models uses: pytorch/test-infra/.github/workflows/macos_job.yml@main @@ -344,6 +287,68 @@ jobs: fi echo "::endgroup::" + prepare-test-specs: + runs-on: linux.2xlarge + needs: + - set-parameters + - export-models + strategy: + matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Prepare the spec + id: prepare + shell: bash + env: + BENCHMARK_CONFIG: ${{ toJSON(matrix) }} + working-directory: extension/benchmark/apple/Benchmark + run: | + set -eux + + # The model will be exported in the next step to this S3 path + MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip" + # Check if the model artifact exists, fail this step skip generating test-spec. + curl -s --head -f ${MODEL_PATH} + # We could write a script to properly use jinja here, but there is only one variable, + # so let's just sed it + sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2 + + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') + # The config for this benchmark runs, we save it in the test spec so that it can be fetched + # later by the upload script + sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2 + + cp default-ios-device-farm-appium-test-spec.yml.j2 default-ios-device-farm-appium-test-spec.yml + # Just print the test spec for debugging + cat default-ios-device-farm-appium-test-spec.yml + + # Save the benchmark configs so that we can use it later in the dashboard + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT + + - name: Upload the spec + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }} + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml + + - name: Update the benchmark configs + uses: seemethere/upload-artifact-s3@v5 + with: + s3-bucket: gha-artifacts + s3-prefix: | + ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ + retention-days: 1 + if-no-files-found: error + path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json + + build-benchmark-app: name: build-benchmark-app uses: pytorch/test-infra/.github/workflows/macos_job.yml@main