From 06f486002deb1729613bc5cf08443b2484d5fd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 3 May 2021 15:36:02 +0200 Subject: [PATCH 1/4] Disable the rust integration tests by default in the docker-compose setup --- .github/workflows/integration.yml | 4 +--- ci/scripts/integration_arrow.sh | 3 --- ci/scripts/rust_build.sh | 19 ++++++++++++++++--- dev/archery/archery/cli.py | 3 ++- docker-compose.yml | 1 + 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fb41f36caf8..59007e0ec8a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -29,7 +29,6 @@ on: - 'cpp/**' - 'java/**' - 'format/**' - - 'rust/**' pull_request: paths: - '.github/workflows/integration.yml' @@ -41,7 +40,6 @@ on: - 'cpp/**' - 'java/**' - 'format/**' - - 'rust/**' env: DOCKER_VOLUME_PREFIX: ".docker/" @@ -82,7 +80,7 @@ jobs: - name: Setup Archery run: pip install -e dev/archery[docker] - name: Execute Docker Build - run: archery docker run conda-integration + run: archery docker run -e ARCHERY_INTEGRATION_WITH_RUST=1 conda-integration - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' continue-on-error: true diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh index 5d2e71916ed..c04b9cd52d9 100755 --- a/ci/scripts/integration_arrow.sh +++ b/ci/scripts/integration_arrow.sh @@ -20,9 +20,6 @@ set -ex arrow_dir=${1} -source_dir=${1}/cpp -build_dir=${2}/cpp - gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration pip install -e $arrow_dir/dev/archery diff --git a/ci/scripts/rust_build.sh b/ci/scripts/rust_build.sh index 726ecd80f1a..237b814091f 100755 --- a/ci/scripts/rust_build.sh +++ b/ci/scripts/rust_build.sh @@ -19,6 +19,7 @@ set -ex +arrow_dir=${1} source_dir=${1}/rust # This file is used to build the rust binaries needed for the @@ -31,6 +32,21 @@ export RUSTFLAGS="-C debuginfo=1" export ARROW_TEST_DATA=${arrow_dir}/testing/data export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data + +if [ "${ARCHERY_INTEGRATION_WITH_RUST}" -eq "0" ]; then + echo "====================================================================" + echo "Not building the Rust implementation." + echo "====================================================================" + exit 0; +elif [ ! -d "${source_dir}" ]; then + echo "====================================================================" + echo "The Rust source is missing from location ${source_dir}, please clone" + echo "the arrow-rs repository before running the integration tests:" + echo " git clone https://github.com/apache/arrow-rs.git ${source_dir}" + echo "====================================================================" + exit 1; +fi + # show activated toolchain rustup show @@ -39,7 +55,4 @@ pushd ${source_dir} # build only the integration testing binaries cargo build -p arrow-integration-testing -# Remove incremental build artifacts to save space -rm -rf target/debug/deps/ target/debug/build/ - popd diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 5329e0abbe2..63396e149a5 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -726,7 +726,8 @@ def _set_default(opt, default): @click.option('--with-go', type=bool, default=False, help='Include Go in integration tests') @click.option('--with-rust', type=bool, default=False, - help='Include Rust in integration tests') + help='Include Rust in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_RUST") @click.option('--write_generated_json', default=False, help='Generate test JSON to indicated path') @click.option('--run-flight', is_flag=True, default=False, diff --git a/docker-compose.yml b/docker-compose.yml index 4a3092ec04d..4158ee3ff64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1197,6 +1197,7 @@ services: <<: *ccache # tell archery where the arrow binaries are located ARROW_CPP_EXE_PATH: /build/cpp/debug + ARCHERY_INTEGRATION_WITH_RUST: 0 command: ["/arrow/ci/scripts/rust_build.sh /arrow /build && /arrow/ci/scripts/cpp_build.sh /arrow /build && From d4fb6da1ab125ddcefd4e19e042ab8171907b7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 3 May 2021 15:41:55 +0200 Subject: [PATCH 2/4] Doesn't know the path outside of the container --- ci/scripts/rust_build.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/ci/scripts/rust_build.sh b/ci/scripts/rust_build.sh index 237b814091f..3532ea3d5c6 100755 --- a/ci/scripts/rust_build.sh +++ b/ci/scripts/rust_build.sh @@ -17,14 +17,14 @@ # specific language governing permissions and limitations # under the License. -set -ex +set -e arrow_dir=${1} source_dir=${1}/rust -# This file is used to build the rust binaries needed for the -# archery integration tests. Testing of the rust implementation -# in normal CI is handled by github workflows +# This file is used to build the rust binaries needed for the archery +# integration tests. Testing of the rust implementation in normal CI is handled +# by github workflows in the arrow-rs repository. # Disable full debug symbol generation to speed up CI build / reduce memory required export RUSTFLAGS="-C debuginfo=1" @@ -32,21 +32,22 @@ export RUSTFLAGS="-C debuginfo=1" export ARROW_TEST_DATA=${arrow_dir}/testing/data export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data - if [ "${ARCHERY_INTEGRATION_WITH_RUST}" -eq "0" ]; then - echo "====================================================================" + echo "=====================================================================" echo "Not building the Rust implementation." - echo "====================================================================" + echo "=====================================================================" exit 0; elif [ ! -d "${source_dir}" ]; then - echo "====================================================================" - echo "The Rust source is missing from location ${source_dir}, please clone" - echo "the arrow-rs repository before running the integration tests:" - echo " git clone https://github.com/apache/arrow-rs.git ${source_dir}" - echo "====================================================================" + echo "=====================================================================" + echo "The Rust source is missing. Please clone the arrow-rs repository" + echo "to arrow/rust before running the integration tests:" + echo " git clone https://github.com/apache/arrow-rs.git path/to/arrow/rust" + echo "=====================================================================" exit 1; fi +set -x + # show activated toolchain rustup show From 574cd6e70fd5b1c1289d3c17dccc02b0c1469523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 3 May 2021 20:22:24 +0200 Subject: [PATCH 3/4] Explicitly enable the implementations --- ci/scripts/integration_arrow.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh index c04b9cd52d9..ecd5c63499b 100755 --- a/ci/scripts/integration_arrow.sh +++ b/ci/scripts/integration_arrow.sh @@ -24,7 +24,14 @@ gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration pip install -e $arrow_dir/dev/archery -archery integration --with-all --run-flight \ +# Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1 +archery integration \ + --run-flight \ + --with-all \ + --with-cpp \ + --with-java \ + --with-js \ + --with-go \ --gold-dirs=$gold_dir/0.14.1 \ --gold-dirs=$gold_dir/0.17.1 \ --gold-dirs=$gold_dir/1.0.0-bigendian \ From 8cad727d14135894813e4b28827e4029a564d45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 3 May 2021 20:46:11 +0200 Subject: [PATCH 4/4] Pass true flags from the CLI --- ci/scripts/integration_arrow.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh index ecd5c63499b..453561d3171 100755 --- a/ci/scripts/integration_arrow.sh +++ b/ci/scripts/integration_arrow.sh @@ -27,11 +27,10 @@ pip install -e $arrow_dir/dev/archery # Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1 archery integration \ --run-flight \ - --with-all \ - --with-cpp \ - --with-java \ - --with-js \ - --with-go \ + --with-cpp=1 \ + --with-java=1 \ + --with-js=1 \ + --with-go=1 \ --gold-dirs=$gold_dir/0.14.1 \ --gold-dirs=$gold_dir/0.17.1 \ --gold-dirs=$gold_dir/1.0.0-bigendian \