diff --git a/TESTING.md b/TESTING.md index 179774638429..941016c3b108 100644 --- a/TESTING.md +++ b/TESTING.md @@ -48,9 +48,6 @@ Bigtable integration tests can either be run against an emulator or a real Bigta target environment can be selected via the `bigtable.env` system property. By default it is set to `emulator` and the other option is `prod`. -To use the `emulator` environment, please install the gcloud sdk and use it to install the -`cbtemulator` via `gcloud components install bigtable`. - To use the `prod` environment: 1. Set up the target table using `google-cloud-bigtable/scripts/setup-test-table.sh` 2. Download the [JSON service account credentials file][create-service-account] from the Google diff --git a/google-cloud-clients/google-cloud-bigtable/pom.xml b/google-cloud-clients/google-cloud-bigtable/pom.xml index a4c3d8893e29..d60d69a0d632 100644 --- a/google-cloud-clients/google-cloud-bigtable/pom.xml +++ b/google-cloud-clients/google-cloud-bigtable/pom.xml @@ -47,6 +47,11 @@ grpc-google-cloud-bigtable-admin-v2 test + + com.google.cloud + google-cloud-bigtable-emulator + test + org.mockito mockito-all diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java deleted file mode 100644 index beef201ef56f..000000000000 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/Emulator.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.it.env; - -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; -import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.BigtableDataSettings; -import com.google.common.io.CharStreams; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.Callable; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** Java wrapper around the gcloud bigtable emulator. */ -// TODO(igorbernstein): Clean this up and externalize this in a separate artifact -// TODO(igorbernstein): Stop depending on gcloud for the binary, instead wrap it in a jar. -class Emulator { - private static final Logger LOGGER = Logger.getLogger(Emulator.class.getName()); - - private final Path executable; - private Process process; - private boolean isStopped = true; - private BigtableTableAdminClient tableAdminClient; - private BigtableDataClient dataClient; - - private static final String PROJECT_ID = "fake-project"; - private static final String INSTANCE_ID = "fake-instance"; - - // Use the gcloud installed emulator - static Emulator createGCloud() { - final Path gcloudSdkPath; - - try { - gcloudSdkPath = getGcloudSdkPath(); - } catch (Exception e) { - throw new RuntimeException("Failed to get the gcloud SDK path. Is it installed?", e); - } - - Path emulatorPath = - gcloudSdkPath.resolve(Paths.get("platform", "bigtable-emulator", "cbtemulator")); - - if (!Files.exists(emulatorPath)) { - throw new RuntimeException( - "cbtemulator is not installed, please install with `gcloud components install bigtable`"); - } - - return new Emulator(emulatorPath); - } - - private Emulator(Path executable) { - this.executable = executable; - } - - void start() throws Exception { - int availablePort = getAvailablePort(); - - process = Runtime.getRuntime().exec(executable + " -port " + "" + availablePort); - pipeStreamToLog(process.getInputStream(), Level.INFO); - pipeStreamToLog(process.getErrorStream(), Level.WARNING); - isStopped = false; - - waitForPort(availablePort); - - tableAdminClient = - BigtableTableAdminClient.create( - BigtableTableAdminSettings.newBuilderForEmulator(availablePort) - .setProjectId(PROJECT_ID) - .setInstanceId(INSTANCE_ID) - .build()); - - dataClient = - BigtableDataClient.create( - BigtableDataSettings.newBuilderForEmulator(availablePort) - .setProjectId(PROJECT_ID) - .setInstanceId(INSTANCE_ID) - .build()); - - Runtime.getRuntime() - .addShutdownHook( - new Thread() { - @Override - public void run() { - if (!isStopped) { - isStopped = true; - process.destroy(); - } - } - }); - } - - void stop() throws Exception { - try { - dataClient.close(); - tableAdminClient.close(); - } finally { - isStopped = true; - process.destroy(); - } - } - - BigtableDataClient getDataClient() { - return dataClient; - } - - BigtableTableAdminClient getTableAdminClient() { - return tableAdminClient; - } - - // - private static Path getGcloudSdkPath() throws Exception { - Process p = Runtime.getRuntime().exec("gcloud info --format=value(installation.sdk_root)"); - pipeStreamToLog(p.getErrorStream(), Level.WARNING); - - String sdkRoot = bufferOutput(p.getInputStream()).get(1, TimeUnit.MINUTES).trim(); - - if (p.waitFor() != 0) { - throw new RuntimeException("Failed to get sdk root, is gcloud sdk installed?"); - } - return Paths.get(sdkRoot); - } - - private static int getAvailablePort() { - try (ServerSocket serverSocket = new ServerSocket(0)) { - return serverSocket.getLocalPort(); - } catch (IOException e) { - throw new RuntimeException("Failed to find open port"); - } - } - - private void waitForPort(int port) throws InterruptedException, TimeoutException { - for (int i = 0; i < 100; i++) { - try (Socket ignored = new Socket("localhost", port)) { - return; - } catch (IOException e) { - Thread.sleep(200); - } - } - - throw new TimeoutException("Timed out waiting for server to start"); - } - - private static void pipeStreamToLog(final InputStream stream, final Level level) { - final BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); - - Thread thread = - new Thread( - new Runnable() { - @Override - public void run() { - try { - String line; - while ((line = reader.readLine()) != null) { - LOGGER.log(level, line); - } - } catch (IOException e) { - LOGGER.log(Level.WARNING, "Failed to read process stream", e); - } - } - }); - thread.setDaemon(true); - thread.start(); - } - - private static Future bufferOutput(final InputStream stream) { - FutureTask task = - new FutureTask<>( - new Callable() { - @Override - public String call() throws Exception { - return CharStreams.toString(new InputStreamReader(stream)); - } - }); - task.run(); - - return task; - } - // -} diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java index fbe357c8fcb5..d6abb9ca5ae3 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java @@ -15,8 +15,12 @@ */ package com.google.cloud.bigtable.data.v2.it.env; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.emulator.v2.Emulator; public class EmulatorEnv implements TestEnv { private static final String PROJECT_ID = "fake-project"; @@ -25,19 +29,28 @@ public class EmulatorEnv implements TestEnv { private static final String FAMILY_ID = "cf"; private Emulator emulator; + private BigtableTableAdminClient tableAdminClient; + private BigtableDataClient dataClient; @Override public void start() throws Exception { - emulator = Emulator.createGCloud(); + emulator = Emulator.createBundled(); emulator.start(); - emulator - .getTableAdminClient() - .createTable(CreateTableRequest.of(TABLE_ID).addFamily(FAMILY_ID)); + tableAdminClient = + BigtableTableAdminClient.create( + BigtableTableAdminSettings.newBuilderForEmulator(emulator.getPort()).build()); + dataClient = + BigtableDataClient.create( + BigtableDataSettings.newBuilderForEmulator(emulator.getPort()).build()); + + tableAdminClient.createTable(CreateTableRequest.of(TABLE_ID).addFamily(FAMILY_ID)); } @Override public void stop() throws Exception { + tableAdminClient.close(); + dataClient.close(); emulator.stop(); } @@ -63,7 +76,7 @@ public String getRowPrefix() { @Override public BigtableDataClient getDataClient() { - return emulator.getDataClient(); + return dataClient; } @Override diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java index 240a77ea1a2a..85d743d9c390 100644 --- a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java +++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java @@ -42,11 +42,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.Lists; import org.apache.commons.io.FileUtils; import org.apache.maven.execution.MavenSession; import org.apache.maven.plugin.AbstractMojo; @@ -83,7 +89,20 @@ public class DownloadComponentsMojo extends AbstractMojo { @Parameter(defaultValue = "${session}", readonly = true) private MavenSession session; + private ExecutorService executor; + public void execute() throws MojoExecutionException { + executor = Executors.newCachedThreadPool(); + + try { + executeInner(); + } finally { + executor.shutdown(); + executor = null; + } + } + + private void executeInner() throws MojoExecutionException { if (shouldSkipDownload) { return; } @@ -119,18 +138,36 @@ public void execute() throws MojoExecutionException { checksums = new HashMap<>(); } - // Download any updated components - for (Component component : components) { + // Download any updated components in parallel + List> futures = Lists.newArrayList(); + + for (final Component component : components) { if (!forceRefresh && component.getChecksum().equals(checksums.get(component.getId()))) { continue; } + futures.add(downloadComponentAsync(component)); + } + + // Wait for all downloads to finish and unwrap any errors + for (Future future : futures) { try { - downloadComponent(component); - } catch (Exception e) { - throw new MojoExecutionException("Failed to download component " + component.getId(), e); + future.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof MojoExecutionException) { + throw ((MojoExecutionException) e.getCause()); + } else { + throw new MojoExecutionException( + "Unexpected execution error downloading component", e.getCause()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new MojoExecutionException("Interrupted while downloading component"); } } + if (futures.size() > 0) { + getLog().info("Finished downloading all components"); + } // Write the checksums of the newly updated components. try { @@ -251,6 +288,22 @@ private Map parseLocalChecksums() throws IOException { return results; } + private Future downloadComponentAsync(final Component component) { + return executor.submit( + new Callable() { + @Override + public Void call() throws MojoExecutionException { + try { + downloadComponent(component); + } catch (Exception e) { + throw new MojoExecutionException("Failed to download " + component.getId(), e); + } + + return null; + } + }); + } + /** Downloads and extracts the component into the destinationDir. */ private void downloadComponent(Component component) throws IOException, NoSuchAlgorithmException { getLog().info("Downloading " + component.getId());