Support enumeration of filesystems when credentials are not available (#1189)

Author: jean-philippe-martin

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java

@@ -85,10 +85,11 @@
 @AutoService(FileSystemProvider.class)
 public final class CloudStorageFileSystemProvider extends FileSystemProvider {
 
-  private final Storage storage;
+  private Storage storage;
+  private StorageOptions storageOptions;
 
   // used only when we create a new instance of CloudStorageFileSystemProvider.
-  private static StorageOptions storageOptions;
+  private static StorageOptions futureStorageOptions;
 
   private static class LazyPathIterator extends AbstractIterator<Path> {
     private final Iterator<Blob> blobIterator;
@@ -122,8 +123,8 @@ protected Path computeNext() {
    * Sets options that are only used by the constructor.
    */
   @VisibleForTesting
-  public static void setGCloudOptions(StorageOptions newStorageOptions) {
-    storageOptions = newStorageOptions;
+  public static void setStorageOptions(StorageOptions newStorageOptions) {
+    futureStorageOptions = newStorageOptions;
   }
 
   /**
@@ -133,14 +134,24 @@ public static void setGCloudOptions(StorageOptions newStorageOptions) {
    * @see CloudStorageFileSystem#forBucket(String)
    */
   public CloudStorageFileSystemProvider() {
-    this(storageOptions);
+    this(futureStorageOptions);
   }
 
   CloudStorageFileSystemProvider(@Nullable StorageOptions gcsStorageOptions) {
-    if (gcsStorageOptions == null) {
+    this.storageOptions = gcsStorageOptions;
+
+  }
+
+  // Initialize this.storage, once. This may throw an exception if default authentication
+  // credentials are not available (hence not doing it in the ctor).
+  private void initStorage() {
+    if (this.storage != null) {
+      return;
+    }
+    if (storageOptions == null) {
       this.storage = StorageOptions.defaultInstance().service();
     } else {
-      this.storage = gcsStorageOptions.service();
+      this.storage = storageOptions.service();
     }
   }
 
@@ -154,6 +165,7 @@ public String getScheme() {
    */
   @Override
   public CloudStorageFileSystem getFileSystem(URI uri) {
+    initStorage();
     return newFileSystem(uri, Collections.<String, Object>emptyMap());
   }
 
@@ -186,11 +198,13 @@ && isNullOrEmpty(uri.getUserInfo()),
         "GCS FileSystem URIs mustn't have: port, userinfo, path, query, or fragment: %s",
         uri);
     CloudStorageUtil.checkBucket(uri.getHost());
+    initStorage();
     return new CloudStorageFileSystem(this, uri.getHost(), CloudStorageConfiguration.fromMap(env));
   }
 
   @Override
   public CloudStoragePath getPath(URI uri) {
+    initStorage();
     return CloudStoragePath.getPath(
         getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
   }
@@ -199,6 +213,7 @@ public CloudStoragePath getPath(URI uri) {
   public SeekableByteChannel newByteChannel(
       Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
     checkNotNull(path);
+    initStorage();
     CloudStorageUtil.checkNotNullArray(attrs);
     if (options.contains(StandardOpenOption.WRITE)) {
       // TODO: Make our OpenOptions implement FileAttribute. Also remove buffer option.
@@ -210,6 +225,7 @@ public SeekableByteChannel newByteChannel(
 
   private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption> options)
       throws IOException {
+    initStorage();
     for (OpenOption option : options) {
       if (option instanceof StandardOpenOption) {
         switch ((StandardOpenOption) option) {
@@ -244,7 +260,7 @@ private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption>
 
   private SeekableByteChannel newWriteChannel(Path path, Set<? extends OpenOption> options)
       throws IOException {
-
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
       throw new CloudStoragePseudoDirectoryException(cloudPath);
@@ -318,6 +334,7 @@ private SeekableByteChannel newWriteChannel(Path path, Set<? extends OpenOption>
 
   @Override
   public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
+    initStorage();
     InputStream result = super.newInputStream(path, options);
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     int blockSize = cloudPath.getFileSystem().config().blockSize();
@@ -331,6 +348,7 @@ public InputStream newInputStream(Path path, OpenOption... options) throws IOExc
 
   @Override
   public boolean deleteIfExists(Path path) throws IOException {
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
       throw new CloudStoragePseudoDirectoryException(cloudPath);
@@ -340,6 +358,7 @@ public boolean deleteIfExists(Path path) throws IOException {
 
   @Override
   public void delete(Path path) throws IOException {
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (!deleteIfExists(cloudPath)) {
       throw new NoSuchFileException(cloudPath.toString());
@@ -348,6 +367,7 @@ public void delete(Path path) throws IOException {
 
   @Override
   public void move(Path source, Path target, CopyOption... options) throws IOException {
+    initStorage();
     for (CopyOption option : options) {
       if (option == StandardCopyOption.ATOMIC_MOVE) {
         throw new AtomicMoveNotSupportedException(
@@ -362,6 +382,7 @@ public void move(Path source, Path target, CopyOption... options) throws IOExcep
 
   @Override
   public void copy(Path source, Path target, CopyOption... options) throws IOException {
+    initStorage();
     boolean wantCopyAttributes = false;
     boolean wantReplaceExisting = false;
     boolean setContentType = false;
@@ -492,6 +513,7 @@ public boolean isHidden(Path path) {
 
   @Override
   public void checkAccess(Path path, AccessMode... modes) throws IOException {
+    initStorage();
     for (AccessMode mode : modes) {
       switch (mode) {
         case READ:
@@ -520,6 +542,7 @@ public <A extends BasicFileAttributes> A readAttributes(
     if (type != CloudStorageFileAttributes.class && type != BasicFileAttributes.class) {
       throw new UnsupportedOperationException(type.getSimpleName());
     }
+    initStorage();
     CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
     if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) {
       @SuppressWarnings("unchecked")
@@ -574,6 +597,7 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
   public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
     final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
     checkNotNull(filter);
+    initStorage();
     String prefix = cloudPath.toString();
     final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
         Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
@@ -621,6 +645,7 @@ public int hashCode() {
 
   @Override
   public String toString() {
+    initStorage();
     return MoreObjects.toStringHelper(this).add("storage", storage).toString();
   }
 

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java

@@ -51,7 +51,7 @@ public class CloudStorageFileAttributeViewTest {
 
   @Before
   public void before() {
-    CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
+    CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
     path = Paths.get(URI.create("gs://red/water"));
   }
 

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributesTest.java

@@ -47,7 +47,7 @@ public class CloudStorageFileAttributesTest {
 
   @Before
   public void before() {
-    CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
+    CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
     path = Paths.get(URI.create("gs://bucket/randompath"));
     dir = Paths.get(URI.create("gs://bucket/randompath/"));
   }

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java

@@ -89,7 +89,7 @@ public class CloudStorageFileSystemProviderTest {
 
   @Before
   public void before() {
-    CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
+    CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
   }
 
   @Test
@@ -628,7 +628,7 @@ public void testNullness() throws IOException, NoSuchMethodException, SecurityEx
       tester.setDefault(Path.class, fs.getPath("and/one"));
       tester.setDefault(OpenOption.class, CREATE);
       tester.setDefault(CopyOption.class, COPY_ATTRIBUTES);
-      // can't do that, setGCloudOptions accepts a null argument.
+      // can't do that, setStorageOptions accepts a null argument.
       // TODO(jart): Figure out how to re-enable this.
       // tester.testAllPublicStaticMethods(CloudStorageFileSystemProvider.class);
       tester.testAllPublicInstanceMethods(new CloudStorageFileSystemProvider());
@@ -651,4 +651,4 @@ private static CloudStorageConfiguration permitEmptyPathComponents(boolean value
   private static CloudStorageConfiguration usePseudoDirectories(boolean value) {
     return CloudStorageConfiguration.builder().usePseudoDirectories(value).build();
   }
-}
+}

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java

@@ -56,7 +56,7 @@ public class CloudStorageFileSystemTest {
 
   @Before
   public void before() {
-    CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
+    CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
   }
 
   @Test

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageLateInitializationTest.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage.contrib.nio;
+
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.net.URI;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link CloudStorageFileSystemProvider} late initialization.
+ */
+@RunWith(JUnit4.class)
+public class CloudStorageLateInitializationTest {
+
+  @Rule public final ExpectedException thrown = ExpectedException.none();
+
+  private StorageOptions mockOptions;
+
+  @Before
+  public void before() {
+    mockOptions = mock(StorageOptions.class);
+    Storage mockStorage = mock(Storage.class);
+    when(mockOptions.service()).thenReturn(mockStorage);
+    CloudStorageFileSystemProvider.setStorageOptions(mockOptions);
+  }
+
+  @Test
+  public void ctorDoesNotCreateStorage() {
+    new CloudStorageFileSystemProvider();
+    verify(mockOptions, never()).service();
+  }
+
+  @Test
+  public void getPathCreatesStorageOnce() {
+    CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider();
+    provider.getPath(URI.create("gs://bucket1/wat"));
+    provider.getPath(URI.create("gs://bucket2/wat"));
+    verify(mockOptions, times(1)).service();
+  }
+
+  @Test
+  public void getFileSystemCreatesStorageOnce() {
+    CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider();
+    provider.getFileSystem(URI.create("gs://bucket1"));
+    provider.getFileSystem(URI.create("gs://bucket2"));
+    verify(mockOptions, times(1)).service();
+  }
+
+}

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageOptionsTest.java

@@ -41,7 +41,7 @@ public class CloudStorageOptionsTest {
 
   @Before
   public void before() {
-    CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
+    CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
   }
 
   @Test

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStoragePathTest.java

@@ -46,7 +46,7 @@ public class CloudStoragePathTest {
 
   @Before
   public void before() {
-    CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options());
+    CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.options());
   }
 
   @Test