From 182a5e454e2d8072ef03b0dac45859b04a072557 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Mon, 12 Sep 2022 17:47:39 -0500 Subject: [PATCH 1/3] Migrate SQL and MongoDBIT Postcommit Jobs from Jenkins to GA --- .../job-postcommit-python-mongodbit.yml | 45 ++++++++++++++++++ .github/workflows/job-postcommit-sql.yml | 46 +++++++++++++++++++ .test-infra/jenkins/README.md | 2 - .../job_PostCommit_Python_MongoDBIO_IT.groovy | 38 --------------- .test-infra/jenkins/job_PostCommit_SQL.groovy | 45 ------------------ CI.md | 12 +++++ 6 files changed, 103 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/job-postcommit-python-mongodbit.yml create mode 100644 .github/workflows/job-postcommit-sql.yml delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_SQL.groovy diff --git a/.github/workflows/job-postcommit-python-mongodbit.yml b/.github/workflows/job-postcommit-python-mongodbit.yml new file mode 100644 index 000000000000..4c49c3207729 --- /dev/null +++ b/.github/workflows/job-postcommit-python-mongodbit.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the integration test of python mongodbio class. + +name: Python MongoDBIO IT + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + python_mongodbit: + name: Python MongoDBIT + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:direct:mongodbioIT + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:mongodbioIT \ No newline at end of file diff --git a/.github/workflows/job-postcommit-sql.yml b/.github/workflows/job-postcommit-sql.yml new file mode 100644 index 000000000000..0863bd590b91 --- /dev/null +++ b/.github/workflows/job-postcommit-sql.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the Java SQL postcommit tests, including the suite of integration test + +name: SQL + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + sql: + name: SQL Tests + runs-on: self-hosted + timeout-minutes: 240 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sqlPostCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sqlPostCommit \ No newline at end of file diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c252a4d55aa..fdb415e62372 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -122,14 +122,12 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Python_Examples_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark_PR/) | `Run Python Examples_Spark` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/) | | beam_PostCommit_Python_Chicago_Taxi_Example_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow_PR/) | `Run Chicago Taxi on Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow) | | beam_PostCommit_Python_Chicago_Taxi_Example_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink_PR/) | `Run Chicago Taxi on Flink` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink) | -| beam_PostCommit_Python_MongoDBIO_IT | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT_PR/) | `Run Python MongoDBIO_IT` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT) | | beam_PostCommit_Python_VR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/) | `Run Python Spark ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark) | | beam_PostCommit_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python37_PR/) | `Run Python 3.7 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python37) | | beam_PostCommit_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python38_PR/) | `Run Python 3.8 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python38) | | beam_PostCommit_Sickbay_Python36 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python36_PR/) | `Run Python 3.6 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36) | | beam_PostCommit_Sickbay_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python37_PR/) | `Run Python 3.7 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37) | | beam_PostCommit_Sickbay_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python38_PR/) | `Run Python 3.8 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38) | -| beam_PostCommit_SQL | [cron](https://ci-beam.apache.org/job/beam_PostCommit_SQL/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SQL_PR/) | `Run SQL PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_SQL/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_SQL) | | beam_PostCommit_Website_Publish | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/) | N/A | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish) | | beam_PostCommit_Website_Test | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/) | `Run Full Website Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test) | diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy deleted file mode 100644 index 437d6ab9613c..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the integration test of python mongodbio class. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_MongoDBIO_IT', - 'Run Python MongoDBIO_IT', 'Python MongoDBIO Integration Test',this) { - description('Runs the Python MongoDBIO Integration Test.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:direct:mongodbioIT') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_SQL.groovy b/.test-infra/jenkins/job_PostCommit_SQL.groovy deleted file mode 100644 index 9207cdbae17d..000000000000 --- a/.test-infra/jenkins/job_PostCommit_SQL.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the Java postcommit tests, including the suite of integration -// tests. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_SQL', 'Run SQL PostCommit', - 'SQL Post Commit Tests', this) { - - description('Runs PostCommit tests for Beam SQL.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sqlPostCommit') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/CI.md b/CI.md index ed75a4dd4433..ca4c14015b3f 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,18 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PostCommit Workflows + +#### SQL - [job-postcommit-sql.yml](.github/workflows/job-postcommit-sql.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-----------|----------------|------------------|-----------------------|---------------|--------------------------| +| SQL Tests | Runs SQL Tests | - | Yes | Yes | - | + +#### Python MongoDBIO IT - [job-postcommit-python-mongodbit.yml](.github/workflows/job-postcommit-python-mongodbit.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------|------------------------|------------------|-----------------------|---------------|--------------------------| +| Python MongoDBIT | Runs Python MongoDB IT | - | Yes | Yes | - | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From cdf28f2c0779e8a68e95576113e7933b7deef846 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Mon, 19 Sep 2022 15:07:21 -0500 Subject: [PATCH 2/3] Add permissions: read-all and minor updates to workflows --- .github/workflows/job-postcommit-python-mongodbit.yml | 5 ++++- .github/workflows/job-postcommit-sql.yml | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-postcommit-python-mongodbit.yml b/.github/workflows/job-postcommit-python-mongodbit.yml index 4c49c3207729..4ffbb3cbe1c3 100644 --- a/.github/workflows/job-postcommit-python-mongodbit.yml +++ b/.github/workflows/job-postcommit-python-mongodbit.yml @@ -25,7 +25,8 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] +permissions: read-all jobs: python_mongodbit: @@ -39,6 +40,8 @@ jobs: submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false - name: Run :sdks:python:test-suites:direct:mongodbioIT uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-sql.yml b/.github/workflows/job-postcommit-sql.yml index 0863bd590b91..e6ecc24e59d8 100644 --- a/.github/workflows/job-postcommit-sql.yml +++ b/.github/workflows/job-postcommit-sql.yml @@ -25,7 +25,8 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] +permissions: read-all jobs: sql: @@ -40,6 +41,11 @@ jobs: submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sqlPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: From 2b36aed29f56efd8ab45b825d1704253527499cc Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Mon, 19 Sep 2022 15:08:19 -0500 Subject: [PATCH 3/3] Testing --- .github/workflows/job-postcommit-python-mongodbit.yml | 1 + .github/workflows/job-postcommit-sql.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/job-postcommit-python-mongodbit.yml b/.github/workflows/job-postcommit-python-mongodbit.yml index 4ffbb3cbe1c3..c8b8fcce08bd 100644 --- a/.github/workflows/job-postcommit-python-mongodbit.yml +++ b/.github/workflows/job-postcommit-python-mongodbit.yml @@ -26,6 +26,7 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] + pull_request: permissions: read-all jobs: diff --git a/.github/workflows/job-postcommit-sql.yml b/.github/workflows/job-postcommit-sql.yml index e6ecc24e59d8..55b8bc22e72a 100644 --- a/.github/workflows/job-postcommit-sql.yml +++ b/.github/workflows/job-postcommit-sql.yml @@ -26,6 +26,7 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] + pull_request: permissions: read-all jobs: