Skip to content

Commit 16c5fdf

Browse files
chensuyuechensuyue
authored
Enable microservice docker images auto build and push (#202)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent dfdd08c commit 16c5fdf

6 files changed

Lines changed: 197 additions & 47 deletions

File tree

‎.github/workflows/image-build-on-push.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
# Test
4+
name: Build latest images on push event
5+
6+
on:
7+
push:
8+
branches: ["main"]
9+
paths:
10+
- comps/**
11+
- "!**.md"
12+
- "!**.txt"
13+
- .github/workflows/image-build-on-push.yml
14+
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.ref }}-on-push
17+
cancel-in-progress: true
18+
19+
jobs:
20+
job1:
21+
uses: ./.github/workflows/reuse-get-test-matrix.yml
22+
23+
image-build:
24+
needs: job1
25+
strategy:
26+
matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
27+
uses: ./.github/workflows/reuse-image-build.yml
28+
with:
29+
micro_service: "${{ matrix.service }}"

‎.github/workflows/microservice-test.yml‎

Lines changed: 6 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -20,47 +20,7 @@ concurrency:
2020

2121
jobs:
2222
job1:
23-
name: Get-test-matrix
24-
runs-on: ubuntu-latest
25-
outputs:
26-
run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}
27-
steps:
28-
- name: Checkout out Repo
29-
uses: actions/checkout@v4
30-
with:
31-
ref: "refs/pull/${{ github.event.number }}/merge"
32-
fetch-depth: 0
33-
- name: Get test matrix
34-
id: get-test-matrix
35-
run: |
36-
set -xe
37-
merged_commit=$(git log -1 --format='%H')
38-
changed_files="$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${merged_commit} \
39-
| grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true
40-
services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u)
41-
path_level_1=("asr" "tts")
42-
path_level_3=("llms_summarization" "llms_text-generation" "dataprep_redis")
43-
run_matrix="{\"include\":["
44-
for service in ${services}; do
45-
hardware="gaudi" # default hardware, set based on the changed files
46-
if [[ "${path_level_1[@]}" =~ "${service}" ]]; then
47-
run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
48-
else
49-
vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile' | sort -u)
50-
for vendor in ${vendors}; do
51-
if [[ "${path_level_3[@]}" =~ "${service}_${vendor}" ]]; then
52-
sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u)
53-
for sub_vendor in ${sub_vendors}; do
54-
run_matrix="${run_matrix}{\"service\":\"${service}_${vendor}_${sub_vendor}\",\"hardware\":\"${hardware}\"},"
55-
done
56-
else
57-
run_matrix="${run_matrix}{\"service\":\"${service}_${vendor}\",\"hardware\":\"${hardware}\"},"
58-
fi
59-
done
60-
fi
61-
done
62-
run_matrix=$run_matrix"]}"
63-
echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
23+
uses: ./.github/workflows/reuse-get-test-matrix.yml
6424

6525
Microservice-test:
6626
needs: job1
@@ -80,16 +40,15 @@ jobs:
8040
- name: Run microservice test
8141
env:
8242
HF_TOKEN: ${{ secrets.HF_TOKEN }}
83-
service: ${{ matrix.service }}
43+
service_path: ${{ matrix.service }}
8444
hardware: ${{ matrix.hardware }}
8545
run: |
8646
cd tests
87-
if [ -f test_${service}.sh ]; then timeout 30m bash test_${service}.sh; else echo "Test script not found, skip test!"; fi
47+
service=$(echo $service_path | tr '/' '_')
48+
echo "service=${service}" >> $GITHUB_ENV
49+
if [ -f test_${service}.sh ]; then timeout 10m bash test_${service}.sh; else echo "Test script not found, skip test!"; fi
8850
8951
- name: Clean up container
90-
env:
91-
service: ${{ matrix.service }}
92-
hardware: ${{ matrix.hardware }}
9352
if: cancelled() || failure()
9453
run: |
9554
cid=$(docker ps -aq --filter "name=test-comps-*")
@@ -100,5 +59,5 @@ jobs:
10059
if: ${{ !cancelled() }}
10160
uses: actions/upload-artifact@v4
10261
with:
103-
name: ${{ matrix.service }}-${{ matrix.hardware }}
62+
name: ${{ env.service }}
10463
path: ${{ github.workspace }}/tests/*.log

‎.github/workflows/reuse-get-test-matrix.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# Support push and pull_request events
5+
name: Get Test Matrix
6+
permissions: read-all
7+
on:
8+
workflow_call:
9+
outputs:
10+
run_matrix:
11+
description: "The matrix string"
12+
value: ${{ jobs.job1.outputs.run_matrix }}
13+
14+
jobs:
15+
job1:
16+
name: Get-test-matrix
17+
runs-on: ubuntu-latest
18+
outputs:
19+
run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}
20+
steps:
21+
- name: Get checkout ref
22+
run: |
23+
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
24+
echo "CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge" >> $GITHUB_ENV
25+
else
26+
echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
27+
fi
28+
echo "checkout ref ${{ env.CHECKOUT_REF }}"
29+
30+
- name: Checkout out Repo
31+
uses: actions/checkout@v4
32+
with:
33+
ref: ${{ env.CHECKOUT_REF }}
34+
fetch-depth: 0
35+
36+
- name: Get test matrix
37+
id: get-test-matrix
38+
run: |
39+
set -xe
40+
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
41+
base_commit=${{ github.event.pull_request.base.sha }}
42+
else
43+
base_commit=$(git rev-parse HEAD~1) # push event
44+
fi
45+
merged_commit=$(git log -1 --format='%H')
46+
47+
changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \
48+
grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true
49+
services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u)
50+
path_level_1=("asr" "tts")
51+
path_level_3=("llms/summarization" "llms/text-generation" "dataprep/redis" "retrievers/langchain/redis")
52+
run_matrix="{\"include\":["
53+
for service in ${services}; do
54+
hardware="gaudi" # default hardware, set based on the changed files
55+
if [[ "${path_level_1[@]}" =~ "${service}" ]]; then
56+
run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
57+
else
58+
vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md' | sort -u)
59+
for vendor in ${vendors}; do
60+
if [[ "${path_level_3[@]}" =~ "${service}/${vendor}" ]]; then
61+
sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u)
62+
for sub_vendor in ${sub_vendors}; do
63+
run_matrix="${run_matrix}{\"service\":\"${service}/${vendor}/${sub_vendor}\",\"hardware\":\"${hardware}\"},"
64+
done
65+
else
66+
run_matrix="${run_matrix}{\"service\":\"${service}/${vendor}\",\"hardware\":\"${hardware}\"},"
67+
fi
68+
done
69+
fi
70+
done
71+
run_matrix=$run_matrix"]}"
72+
echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT

‎.github/workflows/reuse-image-build.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Image Build
5+
permissions: read-all
6+
on:
7+
workflow_call:
8+
inputs:
9+
micro_service:
10+
required: true
11+
type: string
12+
13+
jobs:
14+
micro-image-build:
15+
continue-on-error: true
16+
strategy:
17+
matrix:
18+
node: [docker-build-xeon, docker-build-gaudi]
19+
runs-on: ${{ matrix.node }}
20+
steps:
21+
- name: Checkout out Repo
22+
uses: actions/checkout@v4
23+
with:
24+
fetch-depth: 0
25+
26+
- name: Building MicroService Docker Image
27+
id: build-microservice-image
28+
env:
29+
micro_service: ${{ inputs.micro_service }}
30+
run: |
31+
bash .github/workflows/scripts/docker_images_build_push.sh ${micro_service}

‎.github/workflows/scripts/docker_images_build_push.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -xe
6+
7+
WORKSPACE=$PWD
8+
IMAGE_REPO=${IMAGE_REPO:-$OPEA_IMAGE_REPO}
9+
IMAGE_TAG=${IMAGE_TAG:-latest}
10+
11+
function docker_build() {
12+
# docker_build <IMAGE_NAME>
13+
IMAGE_NAME=$1
14+
micro_service=$2
15+
dockerfile_path=${WORKSPACE}/comps/${micro_service}
16+
if [ -f "$dockerfile_path/Dockerfile" ]; then
17+
DOCKERFILE_PATH="$dockerfile_path/Dockerfile"
18+
elif [ -f "$dockerfile_path/docker/Dockerfile" ]; then
19+
DOCKERFILE_PATH="$dockerfile_path/docker/Dockerfile"
20+
else
21+
echo "Dockerfile not found"
22+
exit 1
23+
fi
24+
echo "Building ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG using Dockerfile $DOCKERFILE_PATH"
25+
26+
docker build --no-cache -t ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG -f $DOCKERFILE_PATH .
27+
docker push ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG
28+
docker rmi ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG
29+
}
30+
31+
micro_service=$1
32+
case ${micro_service} in
33+
"asr"|"tts")
34+
IMAGE_NAME="opea/${micro_service}"
35+
;;
36+
"embeddings/langchain")
37+
IMAGE_NAME="opea/embedding-tei"
38+
;;
39+
"retrievers/langchain")
40+
IMAGE_NAME="opea/retriever-redis"
41+
;;
42+
"reranks/langchain")
43+
IMAGE_NAME="opea/reranking-tei"
44+
;;
45+
"llms/text-generation/tgi")
46+
IMAGE_NAME="opea/llm-tgi"
47+
;;
48+
"dataprep/redis/langchain")
49+
IMAGE_NAME="opea/dataprep-redis"
50+
;;
51+
"llms/summarization/tgi")
52+
IMAGE_NAME="opea/llm-docsum-tgi"
53+
;;
54+
*)
55+
echo "Not supported yet"
56+
exit 0
57+
;;
58+
esac
59+
docker_build "${IMAGE_NAME}" "${micro_service}"

tests/test_retrievers_langchain.sh renamed to tests/test_retrievers_langchain_redis.sh

File renamed without changes.

0 commit comments

Comments
 (0)