From 61027835da15745b7b023e1397d8a6fea6450148 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Fri, 31 Jul 2020 16:18:37 -0400 Subject: [PATCH 1/6] Refactors remote file backend. --- .../azure/CloudBlobContainerFileBackend.java | 31 +++++-------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java index e36e3aa8..c9773dae 100644 --- a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java +++ b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java @@ -16,8 +16,6 @@ package gyro.azure; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; @@ -32,6 +30,7 @@ import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.ListBlobItem; import com.psddev.dari.util.ObjectUtils; import com.psddev.dari.util.StringUtils; @@ -106,35 +105,17 @@ public Stream list() throws Exception { @Override public InputStream openInput(String file) throws Exception { - return container().getBlockBlobReference(prefixed(file)).openInputStream(); + return getBlockBlobReference(file).openInputStream(); } @Override public OutputStream openOutput(String file) throws Exception { - return new ByteArrayOutputStream() { - - public void close() { - try { - container() - .getBlockBlobReference(prefixed(file)) - .uploadFromByteArray(toByteArray(), 0, toByteArray().length); - } catch (StorageException | URISyntaxException | IOException e) { - throw new GyroException(e.getMessage()); - } - } - }; + return getBlockBlobReference(file).openOutputStream(); } @Override public void delete(String file) throws Exception { - // Delete the file only if it exists. - try { - container().getBlockBlobReference(prefixed(file)).delete(); - } catch (StorageException e) { - if (e.getHttpStatusCode() != 404) { - throw e; - } - } + getBlockBlobReference(file).deleteIfExists(); } private Azure client() { @@ -183,4 +164,8 @@ private String removeContainerAndPrefix(String file) { return file; } + + private CloudBlockBlob getBlockBlobReference(String file) throws Exception { + return container().getBlockBlobReference(prefixed(file)); + } } From 47fe9f648f8cb7e45cabcb425652ba7af3b2aff4 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Mon, 3 Aug 2020 10:49:37 -0400 Subject: [PATCH 2/6] Implements exists and copy methods in file backend. --- .../azure/CloudBlobContainerFileBackend.java | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java index c9773dae..33700c60 100644 --- a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java +++ b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java @@ -31,11 +31,13 @@ import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.CopyStatus; import com.microsoft.azure.storage.blob.ListBlobItem; import com.psddev.dari.util.ObjectUtils; import com.psddev.dari.util.StringUtils; import gyro.azure.storage.StorageAccountResource; import gyro.core.FileBackend; +import gyro.core.GyroCore; import gyro.core.GyroException; import gyro.core.Type; import gyro.core.auth.Credentials; @@ -96,11 +98,15 @@ public void setCredentials(String credentials) { @Override public Stream list() throws Exception { - return StreamSupport.stream(container().listBlobs(getPrefix(), true).spliterator(), false) - .map(ListBlobItem::getUri) - .map(URI::getPath) - .filter(f -> f.endsWith(".gyro")) - .map(this::removeContainerAndPrefix); + if (this.equals(GyroCore.getStateBackend(getName()))) { + return StreamSupport.stream(container().listBlobs(getPrefix(), true).spliterator(), false) + .map(ListBlobItem::getUri) + .map(URI::getPath) + .filter(f -> f.endsWith(".gyro")) + .map(this::removeContainerAndPrefix); + } + + return Stream.empty(); } @Override @@ -118,6 +124,34 @@ public void delete(String file) throws Exception { getBlockBlobReference(file).deleteIfExists(); } + @Override + public boolean exists(String file) throws Exception { + return getBlockBlobReference(file).exists(); + } + + @Override + public void copy(String source, String destination) throws Exception { + CloudBlockBlob target = getBlockBlobReference(destination); + target.startCopy(getBlockBlobReference(source)); + + long wait = 0L; + + while (true) { + target.downloadAttributes(); + CopyStatus copyStatus = target.getCopyState().getStatus(); + + if (copyStatus != CopyStatus.PENDING) { + if (copyStatus != CopyStatus.SUCCESS) { + throw new GyroException( + String.format("Copying %s to %s failed: %s", source, destination, copyStatus)); + } + break; + } + wait += 1000L; + Thread.sleep(wait); + } + } + private Azure client() { Credentials credentials = getRootScope().getSettings(CredentialsSettings.class) .getCredentialsByName() From 408dbde5152c655e9740a1c36d822f54542190b5 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Mon, 3 Aug 2020 10:52:36 -0400 Subject: [PATCH 3/6] Fixes to use a snapshot version of gyro in development. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8411852a..6fb49014 100644 --- a/build.gradle +++ b/build.gradle @@ -62,7 +62,7 @@ configurations { def azureSdkVersion = '1.31.0' dependencies { - api 'gyro:gyro-core:0.99.4' + api 'gyro:gyro-core:0.99.5' + (releaseBuild ? '' : '-SNAPSHOT') implementation 'com.psddev:dari-util:3.3.607-xe0f27a' implementation 'com.google.guava:guava:23.0' From 1178929b7db82888f78eb1dabb1547897d98be89 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Fri, 7 Aug 2020 17:45:29 -0400 Subject: [PATCH 4/6] Refactors get container method. --- .../azure/CloudBlobContainerFileBackend.java | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java index 33700c60..51b2e1d1 100644 --- a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java +++ b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java @@ -21,6 +21,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.InvalidKeyException; +import java.util.Optional; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -40,7 +41,6 @@ import gyro.core.GyroCore; import gyro.core.GyroException; import gyro.core.Type; -import gyro.core.auth.Credentials; import gyro.core.auth.CredentialsSettings; @Type("cloud-blob-container") @@ -152,26 +152,24 @@ public void copy(String source, String destination) throws Exception { } } - private Azure client() { - Credentials credentials = getRootScope().getSettings(CredentialsSettings.class) - .getCredentialsByName() - .get("azure::" + getCredentials()); - - return AzureResource.createClient((AzureCredentials) credentials); - } - private CloudBlobContainer container() { - StorageAccountResource storage = getRootScope().findResourceById( - StorageAccountResource.class, - getStorageAccount()); - - StorageAccount storageAccount = client().storageAccounts() - .getByResourceGroup(getResourceGroup(), getStorageAccount()); + String account = getStorageAccount(); + StorageAccount storageAccount = Optional.ofNullable(getRootScope()) + .map(e -> e.getSettings(CredentialsSettings.class)) + .map(CredentialsSettings::getCredentialsByName) + .map(e -> e.get("azure::" + getCredentials())) + .filter(AzureCredentials.class::isInstance) + .map(AzureCredentials.class::cast) + .map(AzureResource::createClient) + .map(Azure::storageAccounts) + .map(e -> e.getByResourceGroup(getResourceGroup(), account)) + .orElseThrow(() -> new GyroException("No storage account available!")); + StorageAccountResource storage = getRootScope().findResourceById(StorageAccountResource.class, account); storage.copyFrom(storageAccount); try { - CloudStorageAccount account = CloudStorageAccount.parse(storage.getConnection()); - CloudBlobClient blobClient = account.createCloudBlobClient(); + CloudStorageAccount cloudStorageAccount = CloudStorageAccount.parse(storage.getConnection()); + CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient(); return blobClient.getContainerReference(getCloudBlobContainer()); } catch (StorageException | URISyntaxException | InvalidKeyException ex) { From aea7eeb44dffd8dc14126e54d1031ff78fd0a3f7 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Fri, 7 Aug 2020 17:46:06 -0400 Subject: [PATCH 5/6] Fixes as file backend API changes. --- src/main/java/gyro/azure/CloudBlobContainerFileBackend.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java index 51b2e1d1..3ec50f67 100644 --- a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java +++ b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java @@ -38,6 +38,7 @@ import com.psddev.dari.util.StringUtils; import gyro.azure.storage.StorageAccountResource; import gyro.core.FileBackend; +import gyro.core.FileBackendAccess; import gyro.core.GyroCore; import gyro.core.GyroException; import gyro.core.Type; @@ -115,7 +116,7 @@ public InputStream openInput(String file) throws Exception { } @Override - public OutputStream openOutput(String file) throws Exception { + public OutputStream openOutput(String file, FileBackendAccess acl) throws Exception { return getBlockBlobReference(file).openOutputStream(); } @@ -130,7 +131,7 @@ public boolean exists(String file) throws Exception { } @Override - public void copy(String source, String destination) throws Exception { + public void copy(String source, String destination, FileBackendAccess acl) throws Exception { CloudBlockBlob target = getBlockBlobReference(destination); target.startCopy(getBlockBlobReference(source)); From 8c645db3cd117691c8fccc46103876ee15a3ffa4 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Mon, 10 Aug 2020 16:32:47 -0400 Subject: [PATCH 6/6] Revert "Fixes as file backend API changes." This reverts commit aea7eeb44dffd8dc14126e54d1031ff78fd0a3f7. --- src/main/java/gyro/azure/CloudBlobContainerFileBackend.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java index 3ec50f67..51b2e1d1 100644 --- a/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java +++ b/src/main/java/gyro/azure/CloudBlobContainerFileBackend.java @@ -38,7 +38,6 @@ import com.psddev.dari.util.StringUtils; import gyro.azure.storage.StorageAccountResource; import gyro.core.FileBackend; -import gyro.core.FileBackendAccess; import gyro.core.GyroCore; import gyro.core.GyroException; import gyro.core.Type; @@ -116,7 +115,7 @@ public InputStream openInput(String file) throws Exception { } @Override - public OutputStream openOutput(String file, FileBackendAccess acl) throws Exception { + public OutputStream openOutput(String file) throws Exception { return getBlockBlobReference(file).openOutputStream(); } @@ -131,7 +130,7 @@ public boolean exists(String file) throws Exception { } @Override - public void copy(String source, String destination, FileBackendAccess acl) throws Exception { + public void copy(String source, String destination) throws Exception { CloudBlockBlob target = getBlockBlobReference(destination); target.startCopy(getBlockBlobReference(source));