diff --git a/dropwizard/service/build.gradle.kts b/dropwizard/service/build.gradle.kts index 294279946..21498a592 100644 --- a/dropwizard/service/build.gradle.kts +++ b/dropwizard/service/build.gradle.kts @@ -96,6 +96,7 @@ dependencies { compileOnly(libs.jakarta.annotation.api) compileOnly(libs.spotbugs.annotations) + testImplementation(project(":polaris-tests")) testImplementation(project(":polaris-api-management-model")) testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") @@ -140,6 +141,8 @@ tasks.named("test").configure { if (System.getenv("AWS_REGION") == null) { environment("AWS_REGION", "us-west-2") } + environment("POLARIS_BOOTSTRAP_POLARIS_ROOT_CLIENT_ID", "test-admin") + environment("POLARIS_BOOTSTRAP_POLARIS_ROOT_CLIENT_SECRET", "test-secret") jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") useJUnitPlatform() maxParallelForks = 4 diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java deleted file mode 100644 index 215288dbc..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.catalog; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.util.Map; -import org.apache.iceberg.rest.RESTCatalog; -import org.apache.iceberg.view.ViewCatalogTests; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironment; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog - * client. - */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public abstract class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism - - private RESTCatalog restCatalog; - - @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); - } - - @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm, - TestEnvironment testEnv) { - - Assumptions.assumeFalse(shouldSkip()); - - String userToken = adminToken.token(); - testInfo - .getTestMethod() - .ifPresent( - method -> { - String catalogName = method.getName() + testEnv.testId(); - try (Response response = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - if (response.getStatus() == Response.Status.OK.getStatusCode()) { - // Already exists! Must be in a parameterized test. - // Quick hack to get a unique catalogName. - // TODO: Have a while-loop instead with consecutive incrementing suffixes. - catalogName = catalogName + System.currentTimeMillis(); - } - } - - StorageConfigInfo storageConfig = getStorageConfigInfo(); - String defaultBaseLocation = - storageConfig.getAllowedLocations().getFirst() - + "/" - + System.getenv("USER") - + "/path/to/data"; - - org.apache.polaris.core.admin.model.CatalogProperties props = - org.apache.polaris.core.admin.model.CatalogProperties.builder(defaultBaseLocation) - .addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, - "file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), - "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), - "true") - .build(); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(props) - .setStorageConfigInfo(storageConfig) - .build(); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminToken, - snowmanCredentials, - realm, - catalog, - Map.of()); - }); - } - - /** - * @return The catalog's storage config. - */ - protected abstract StorageConfigInfo getStorageConfigInfo(); - - /** - * @return Whether the tests should be skipped, for example due to environment variables not being - * specified. - */ - protected abstract boolean shouldSkip(); - - @Override - protected RESTCatalog catalog() { - return restCatalog; - } - - @Override - protected org.apache.iceberg.catalog.Catalog tableCatalog() { - return restCatalog; - } - - @Override - protected boolean requiresNamespaceCreate() { - return true; - } - - @Override - protected boolean supportsServerSideRetry() { - return true; - } - - @Override - protected boolean overridesRequestedLocation() { - return true; - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java new file mode 100644 index 000000000..6eeb25ae1 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; + +public class DropwizardApplicationIntegrationTest extends PolarisApplicationIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java new file mode 100644 index 000000000..894d9a74f --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; + +public class DropwizardManagementServiceIntegrationTest + extends PolarisManagementServiceIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java new file mode 100644 index 000000000..5443ef393 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogIntegrationTest; + +public class DropwizardRestCatalogIntegrationTest extends PolarisRestCatalogIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java new file mode 100644 index 000000000..b4cbf1d51 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAwsIntegrationTest; + +public class DropwizardRestCatalogViewAwsIntegrationTest + extends PolarisRestCatalogViewAwsIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java new file mode 100644 index 000000000..248877513 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAzureIntegrationTest; + +public class DropwizardRestCatalogViewAzureIntegrationTest + extends PolarisRestCatalogViewAzureIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java new file mode 100644 index 000000000..885ef4e1f --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; + +public class DropwizardRestCatalogViewFileIntegrationTest + extends PolarisRestCatalogViewFileIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java new file mode 100644 index 000000000..45cde359c --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewGcpIntegrationTest; + +public class DropwizardRestCatalogViewGcpIntegrationTest + extends PolarisRestCatalogViewGcpIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java new file mode 100644 index 000000000..fe4c0b9bd --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.polaris.service.dropwizard.PolarisApplication; +import org.apache.polaris.service.dropwizard.auth.TokenUtils; +import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.Server; +import org.apache.polaris.service.it.ext.PolarisServerManager; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class DropwizardServerManager implements PolarisServerManager { + public static final String TEST_REALM = + "POLARIS"; // referenced in polaris-server-integrationtest.yml + public static final String SERVER_CONFIG_PATH = "polaris-server-integrationtest.yml"; + + @Override + public Server realmForContext(ExtensionContext context) { + try { + return new Holder(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static class Holder implements Server { + private final DropwizardAppExtension ext; + private final AuthToken token; + private final Path logDir; + + public Holder() throws IOException { + logDir = Files.createTempDirectory("polaris-dw-"); + Path logFile = logDir.resolve("application.log").toAbsolutePath(); + + Map config = new HashMap<>(); + // Bind to random port to support parallelism + config.put("server.applicationConnectors[0].port", "0"); + config.put("server.adminConnectors[0].port", "0"); + config.put("logging.appenders[1].type", "file"); + config.put("logging.appenders[1].currentLogFilename", logFile.toString()); + + ConfigOverride[] overrides = + config.entrySet().stream() + .map((e) -> ConfigOverride.config(e.getKey(), e.getValue())) + .toList() + .toArray(new ConfigOverride[0]); + ext = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath(SERVER_CONFIG_PATH), + overrides); + + try { + ext.before(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + token = new AuthToken(getToken(), "root"); + + assertThat(logFile) + .exists() + .content() + .hasSizeGreaterThan(0) + .doesNotContain("ERROR", "FATAL") + .contains("PolarisApplication: Server started successfully"); + } + + private String getToken() { + return TokenUtils.getTokenFromSecrets( + ext.client(), + baseUri().toString(), + adminCredentials().clientId(), + adminCredentials().clientSecret(), + TEST_REALM); + } + + @Override + public String realmId() { + return TEST_REALM; + } + + @Override + public URI baseUri() { + return URI.create(String.format("http://localhost:%d", ext.getLocalPort())); + } + + @Override + public AuthToken adminToken() { + return token; + } + + @Override + public ClientCredentials adminCredentials() { + // These credentials are injected via env. variables from build scripts. + // Cf. POLARIS_BOOTSTRAP_POLARIS_ROOT_CLIENT_ID + return new ClientCredentials("test-admin", "test-secret"); + } + + @Override + public void close() throws IOException { + ext.after(); + FileUtils.deleteDirectory(logDir.toFile()); + } + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java new file mode 100644 index 000000000..26fe3fb70 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisSparkIntegrationTest; + +public class DropwizardSparkIntegrationTest extends PolarisSparkIntegrationTest {} diff --git a/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 000000000..37adc23a1 --- /dev/null +++ b/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.dropwizard.it.DropwizardServerManager \ No newline at end of file diff --git a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml index 9d8f770e5..1f4f2117b 100644 --- a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml +++ b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml @@ -162,3 +162,8 @@ tokenBucketFactory: type: default requestsPerSecond: 9999 windowSeconds: 10 + +# Fake GCP credentials for tests that do not need access to storage +gcp_credentials: + access_token: "abc" + expires_in: 12345 \ No newline at end of file diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index bf7feec16..71490c3cb 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -26,5 +26,6 @@ polaris-service-common=service/common polaris-dropwizard-service=dropwizard/service polaris-eclipselink=extension/persistence/eclipselink polaris-jpa-model=extension/persistence/jpa-model +polaris-tests=integration-tests aggregated-license-report=aggregated-license-report polaris-version=tools/version diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts new file mode 100644 index 000000000..17c0f60cb --- /dev/null +++ b/integration-tests/build.gradle.kts @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { id("polaris-server") } + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-api-management-model")) + + implementation(libs.jakarta.ws.rs.api) + implementation(libs.guava) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + implementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") + implementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") + + implementation(libs.hadoop.common) { + exclude("org.slf4j", "slf4j-reload4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + exclude("org.apache.zookeeper", "zookeeper") + } + + implementation(libs.auth0.jwt) + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + implementation("org.apache.iceberg:iceberg-spark-3.5_2.12") + implementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") + implementation("org.apache.spark:spark-sql_2.12:3.5.1") { + // exclude log4j dependencies + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-api") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + } + + implementation(platform(libs.junit.bom)) + implementation("org.junit.jupiter:junit-jupiter") + compileOnly("org.junit.jupiter:junit-jupiter-engine") + implementation(libs.assertj.core) + implementation(libs.mockito.core) +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/AuthToken.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/AuthToken.java new file mode 100644 index 000000000..d6a9805b0 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/AuthToken.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +public record AuthToken(String token, String principalName) {} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java new file mode 100644 index 000000000..329500334 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public class CatalogApi extends RestApi { + CatalogApi(PolarisApiClient client, AuthToken token, URI uri) { + super(client, token, uri); + } + + public AuthToken obtainToken(PrincipalWithCredentials principal) { + try (Response response = + request("v1/oauth/tokens") + .post( + Entity.form( + new MultivaluedHashMap<>( + Map.of( + "grant_type", + "client_credentials", + "scope", + "PRINCIPAL_ROLE:ALL", + "client_id", + principal.getCredentials().getClientId(), + "client_secret", + principal.getCredentials().getClientSecret()))))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + String token = response.readEntity(OAuthTokenResponse.class).token(); + return new AuthToken(token, principal.getPrincipal().getName()); + } + } + + public void createNamespace(String catalogName, String namespaceName) { + try (Response response = + request("v1/{cat}/namespaces", Map.of("cat", catalogName)) + .post( + Entity.json( + CreateNamespaceRequest.builder() + .withNamespace(Namespace.of(namespaceName)) + .build()))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + } + } + + public List listNamespaces(String catalog) { + try (Response response = request("v1/{cat}/namespaces", Map.of("cat", catalog)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + ListNamespacesResponse res = response.readEntity(ListNamespacesResponse.class); + return res.namespaces(); + } + } + + public void deleteNamespaces(String catalog, Namespace namespace) { + try (Response response = + request( + "v1/{cat}/namespaces/{ns}", + Map.of("cat", catalog, "ns", RESTUtil.encodeNamespace(namespace))) + .delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java new file mode 100644 index 000000000..528fd7497 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +public record ClientCredentials(String clientId, String clientSecret) {} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java new file mode 100644 index 000000000..2e0bd0d61 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static org.apache.polaris.service.it.env.PolarisApiClient.REALM_HEADER; +import static org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest.PRINCIPAL_ROLE_ALL; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.iceberg.catalog.SessionCatalog; +import org.apache.iceberg.rest.HTTPClient; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public final class IcebergHelper { + private IcebergHelper() {} + + public static RESTCatalog restCatalog( + PolarisApiClient client, + PrincipalWithCredentials credentials, + String catalog, + Map extraProperties) { + SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); + RESTCatalog restCatalog = + new RESTCatalog( + context, + (config) -> + HTTPClient.builder(config) + .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) + .build()); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put(org.apache.iceberg.CatalogProperties.URI, client.catalogApiEndpoint().toString()) + .put( + OAuth2Properties.CREDENTIAL, + credentials.getCredentials().getClientId() + + ":" + + credentials.getCredentials().getClientSecret()) + .put(OAuth2Properties.SCOPE, PRINCIPAL_ROLE_ALL) + .put( + org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO") + .put("warehouse", catalog) + .put("header." + REALM_HEADER, client.realm()) + .putAll(extraProperties); + + restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast()); + return restCatalog; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java new file mode 100644 index 000000000..81c616824 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogGrant; +import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CatalogRoles; +import org.apache.polaris.core.admin.model.Catalogs; +import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.GrantResources; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalRoles; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.Principals; +import org.apache.polaris.core.admin.model.UpdateCatalogRequest; + +public class ManagementApi extends RestApi { + ManagementApi(PolarisApiClient client, AuthToken token, URI uri) { + super(client, token, uri); + } + + public PrincipalWithCredentials createPrincipalWithRole(String principalName, String roleName) { + PrincipalWithCredentials credentials = createPrincipal(principalName); + createPrincipalRole(roleName); + assignPrincipalRole(principalName, roleName); + return credentials; + } + + public PrincipalWithCredentials createPrincipal(String name) { + try (Response createPResponse = + request("v1/principals").post(Entity.json(new Principal(name)))) { + assertThat(createPResponse).returns(CREATED.getStatusCode(), Response::getStatus); + return createPResponse.readEntity(PrincipalWithCredentials.class); + } + } + + public void createPrincipalRole(String name) { + createPrincipalRole(new PrincipalRole(name)); + } + + public void createPrincipalRole(PrincipalRole role) { + try (Response createPrResponse = request("v1/principal-roles").post(Entity.json(role))) { + assertThat(createPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void assignPrincipalRole(String principalName, String roleName) { + try (Response assignPrResponse = + request("v1/principals/{prince}/principal-roles", Map.of("prince", principalName)) + .put(Entity.json(new GrantPrincipalRoleRequest(new PrincipalRole(roleName))))) { + assertThat(assignPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void createCatalogRole(String catalogName, String catalogRoleName) { + try (Response response = + request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) + .post(Entity.json(new CatalogRole(catalogRoleName)))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .put(Entity.json(grant))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void grantCatalogRoleToPrincipalRole( + String principalRoleName, String catalogName, CatalogRole catalogRole) { + try (Response response = + request( + "v1/principal-roles/{role}/catalog-roles/{cat}", + Map.of("role", principalRoleName, "cat", catalogName)) + .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public GrantResources listGrants(String catalogName, String catalogRoleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .get()) { + assertThat(response).returns(OK.getStatusCode(), Response::getStatus); + return response.readEntity(GrantResources.class); + } + } + + public void createCatalog(String principalRoleName, Catalog catalog) { + createCatalog(catalog); + + // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS + String catalogRoleName = "custom-admin"; + createCatalogRole(catalog.getName(), catalogRoleName); + + CatalogGrant grantResource = + new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); + Map catalogVars = Map.of("cat", catalog.getName(), "role", catalogRoleName); + try (Response response = + request("v1/catalogs/{cat}/catalog-roles/{role}/grants", catalogVars) + .put(Entity.json(grantResource))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + + CatalogGrant grantAccessResource = + new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); + try (Response response = + request("v1/catalogs/{cat}/catalog-roles/{role}/grants", catalogVars) + .put(Entity.json(grantAccessResource))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + + // Assign this new CatalogRole to the service_admin PrincipalRole + try (Response response = + request( + "v1/principal-roles/{role}/catalog-roles/{cat}", + Map.of("role", principalRoleName, "cat", catalog.getName())) + .put(Entity.json(new CatalogRole(catalogRoleName)))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public void createCatalog(Catalog catalog) { + try (Response response = request("v1/catalogs").post(Entity.json(catalog))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public Catalog getCatalog(String name) { + try (Response response = request("v1/catalogs/{name}", Map.of("name", name)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Catalog.class); + } + } + + public void updateCatalog(Catalog catalog, Map catalogProps) { + try (Response response = + request("v1/catalogs/{name}", Map.of("name", catalog.getName())) + .put( + Entity.json( + new UpdateCatalogRequest( + catalog.getEntityVersion(), + catalogProps, + catalog.getStorageConfigInfo())))) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + } + } + + public void deleteCatalog(String catalogName) { + try (Response response = request("v1/catalogs/{cat}", Map.of("cat", catalogName)).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public CatalogRole getCatalogRole(String catalogName, String roleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of("cat", catalogName, "role", roleName)) + .get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(CatalogRole.class); + } + } + + public List listCatalogRoles(String catalogName) { + try (Response response = + request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(CatalogRoles.class).getRoles(); + } + } + + public List listPrincipals() { + try (Response response = request("v1/principals").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Principals.class).getPrincipals(); + } + } + + public List listPrincipalRoles() { + try (Response response = request("v1/principal-roles").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(PrincipalRoles.class).getRoles(); + } + } + + public List listCatalogs() { + try (Response response = request("v1/catalogs").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Catalogs.class).getCatalogs(); + } + } + + public void deleteCatalogRole(String catalogName, CatalogRole role) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of("cat", catalogName, "role", role.getName())) + .delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void deletePrincipal(Principal principal) { + try (Response response = + request("v1/principals/{name}", Map.of("name", principal.getName())).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void deletePrincipalRole(PrincipalRole role) { + try (Response response = + request("v1/principal-roles/{name}", Map.of("name", role.getName())).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiClient.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiClient.java new file mode 100644 index 000000000..aa10dc28a --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiClient.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import jakarta.ws.rs.client.Client; +import java.net.URI; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public record PolarisApiClient(Client client, String realm, URI baseUri) { + + public static String REALM_HEADER = "realm"; + + public URI catalogApiEndpoint() { + return baseUri.resolve("api/catalog"); + } + + public CatalogApi catalogApi(AuthToken token) { + return new CatalogApi(this, token, catalogApiEndpoint()); + } + + public CatalogApi catalogApi(PrincipalWithCredentials principal) { + return catalogApi(obtainToken(principal)); + } + + public AuthToken obtainToken(PrincipalWithCredentials principal) { + CatalogApi anon = new CatalogApi(this, null, catalogApiEndpoint()); + return anon.obtainToken(principal); + } + + public ManagementApi managementApi(AuthToken token) { + return new ManagementApi(this, token, baseUri.resolve("api/management")); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java new file mode 100644 index 000000000..cb98a3c5f --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; +import java.net.URI; +import java.util.Map; + +public class RestApi { + private final PolarisApiClient client; + private final AuthToken token; + private final URI uri; + + RestApi(PolarisApiClient client, AuthToken token, URI uri) { + this.client = client; + this.token = token; + this.uri = uri; + } + + public Invocation.Builder request(String path) { + return request(path, Map.of()); + } + + public Invocation.Builder request(String path, Map templateValues) { + WebTarget target = client.client().target(uri).path(path); + for (Map.Entry entry : templateValues.entrySet()) { + target = target.resolveTemplate(entry.getKey(), entry.getValue()); + } + Invocation.Builder request = target.request("application/json"); + request = request.header(PolarisApiClient.REALM_HEADER, client.realm()); + if (token != null) { + request = request.header("Authorization", "Bearer " + token.token()); + } + return request; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java new file mode 100644 index 000000000..63594e68d --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import java.net.URI; + +public interface Server extends AutoCloseable { + String realmId(); + + URI baseUri(); + + AuthToken adminToken(); + + ClientCredentials adminCredentials(); +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java new file mode 100644 index 000000000..03dae5880 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import static java.util.concurrent.TimeUnit.*; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.ext.ContextResolver; +import java.util.ServiceLoader; +import org.apache.iceberg.rest.RESTSerializers; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.PolarisApiClient; +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.ExtensionContext.Store.CloseableResource; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.platform.engine.UniqueId; + +public class PolarisIntegrationTestExtension implements ParameterResolver { + private static final Namespace NAMESPACE = + Namespace.create(PolarisIntegrationTestExtension.class); + + private static final PolarisServerManager manager = + ServiceLoader.load(PolarisServerManager.class) + .findFirst() + .orElseThrow(() -> new IllegalStateException("PolarisServerManager not found")); + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + Class type = parameterContext.getParameter().getType(); + return type.isAssignableFrom(PolarisApiClient.class) + || type.isAssignableFrom(ClientCredentials.class) + || type.isAssignableFrom(AuthToken.class); + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + Env env = env(extensionContext); + Class type = parameterContext.getParameter().getType(); + if (type.isAssignableFrom(PolarisApiClient.class)) { + return env.client(); + } else if (type.isAssignableFrom(ClientCredentials.class)) { + return env.server.adminCredentials(); + } else if (type.isAssignableFrom(AuthToken.class)) { + return env.server.adminToken(); + } + throw new IllegalStateException("Unable to resolve parameter: " + parameterContext); + } + + private Env env(ExtensionContext context) { + ExtensionContext classCtx = classContext(context); + ExtensionContext.Store store = classCtx.getStore(NAMESPACE); + return store.getOrComputeIfAbsent( + Env.class, (key) -> new Env(manager.realmForContext(classCtx)), Env.class); + } + + private ExtensionContext classContext(ExtensionContext context) { + while (context.getParent().isPresent()) { + UniqueId id = UniqueId.parse(context.getUniqueId()); + if ("class".equals(id.getLastSegment().getType())) { + break; + } + + context = context.getParent().get(); + } + + return context; + } + + private static class Env implements CloseableResource { + private final Server server; + private final Client client; + + private Env(Server server) { + this.server = server; + + ObjectMapper mapper = new ObjectMapper(); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(mapper); + + // Note: converting to lambda apparently breaks discovery/injections in the client below + //noinspection Convert2Lambda + ContextResolver mapperProvider = + new ContextResolver<>() { + @Override + public ObjectMapper getContext(Class type) { + return mapper; + } + }; + + this.client = + ClientBuilder.newBuilder() + .readTimeout(5, MINUTES) + .connectTimeout(1, MINUTES) + .register(mapperProvider) + .build(); + } + + PolarisApiClient client() { + return new PolarisApiClient(client, server.realmId(), server.baseUri()); + } + + @Override + public void close() throws Throwable { + server.close(); + client.close(); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java new file mode 100644 index 000000000..3cb4f72a4 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; + +public interface PolarisServerManager { + + /** + * Returns server connection parameters for the tests under the specified context. + * + *

Implementations may reuse the same server for multiple contexts or create a fresh one for + * each context. In any case, {@link Server#close()} will be invoked when the context provided as + * the argument to this call is closed. + */ + Server realmForContext(ExtensionContext context); +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java similarity index 70% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index af2013fb0..b2da34afe 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -16,30 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE; +import static org.apache.polaris.service.it.env.PolarisApiClient.REALM_HEADER; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.ProcessingException; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.function.Supplier; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; @@ -81,14 +74,11 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse; -import org.assertj.core.api.Assertions; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.PolarisApiClient; +import org.apache.polaris.service.it.env.RestApi; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -96,120 +86,61 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public class PolarisApplicationIntegrationTest { - @TempDir private static Path tempDir; - private static final Supplier CURRENT_LOG = - () -> tempDir.resolve("application.log").toString(); - private static final Logger LOGGER = - LoggerFactory.getLogger(PolarisApplicationIntegrationTest.class); +@ExtendWith(PolarisIntegrationTestExtension.class) +public class PolarisApplicationIntegrationTest { public static final String PRINCIPAL_ROLE_NAME = "admin"; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0"), // Bind to random port to support parallelism - ConfigOverride.config("logging.appenders[1].type", "file"), - ConfigOverride.config("logging.appenders[1].currentLogFilename", CURRENT_LOG)); + public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL"; private static String userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; private static Path testDir; private static String realm; + private static RestApi managementApi; + private static PolarisApiClient apiClient; + private static ClientCredentials clientCredentials; + @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String polarisRealm) + public static void setup(PolarisApiClient client, AuthToken token, ClientCredentials credentials) throws IOException { - realm = polarisRealm; - - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .contains("PolarisApplication: Server started successfully"); + apiClient = client; + realm = client.realm(); + clientCredentials = credentials; testDir = Path.of("build/test_data/iceberg/" + realm); FileUtils.deleteQuietly(testDir.toFile()); Files.createDirectories(testDir); - PolarisApplicationIntegrationTest.userToken = userToken.token(); - PolarisApplicationIntegrationTest.snowmanCredentials = snowmanCredentials; + userToken = token.token(); + managementApi = client.managementApi(token); PrincipalRole principalRole = new PrincipalRole(PRINCIPAL_ROLE_NAME); try (Response createPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principalRole))) { + managementApi.request("v1/principal-roles").post(Entity.json(principalRole))) { assertThat(createPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response assignPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principals/%s/principal-roles", - EXT.getLocalPort(), snowmanCredentials.identifier().principalName())) - .request("application/json") - .header("Authorization", "Bearer " + PolarisApplicationIntegrationTest.userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request("v1/principals/{name}/principal-roles", Map.of("name", token.principalName())) .put(Entity.json(principalRole))) { assertThat(assignPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - - assertZeroErrorsInApplicationLog(); } @AfterAll public static void deletePrincipalRole() { - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s", - EXT.getLocalPort(), PRINCIPAL_ROLE_NAME)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request("v1/principal-roles/{role}", Map.of("role", PRINCIPAL_ROLE_NAME)) .delete() .close(); } - private static void assertZeroErrorsInApplicationLog() { - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .hasSizeGreaterThan(0) - .doesNotContain("ERROR", "FATAL"); - } - /** * Create a new catalog for each test case. Assign the snowman catalog-admin principal role the * admin role of the new catalog. - * - * @param testInfo */ @BeforeEach public void before(TestInfo testInfo) { @@ -264,40 +195,24 @@ private static void createCatalog( .setProperties(props) .setStorageConfigInfo(storageConfig) .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalog))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s", - EXT.getLocalPort(), - catalogName, - PolarisEntityConstants.getNameOfCatalogAdminRole())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of( + "cat", catalogName, "role", PolarisEntityConstants.getNameOfCatalogAdminRole())) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s/catalog-roles/%s", - EXT.getLocalPort(), principalRoleName, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request( + "v1/principal-roles/{prin-role}/catalog-roles/{cat}", + Map.of("cat", catalogName, "prin-role", principalRoleName)) .put(Entity.json(catalogRole))) { assertThat(assignResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -311,14 +226,14 @@ private static RESTSessionCatalog newSessionCatalog(String catalog) { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + apiClient.catalogApiEndpoint().toString(), OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", catalog, - "header." + REALM_PROPERTY_KEY, + "header." + REALM_HEADER, realm)); return sessionCatalog; } @@ -333,7 +248,7 @@ public void testIcebergListNamespaces() throws IOException { } @Test - public void testConfigureCatalogCaseSensitive() throws IOException { + public void testConfigureCatalogCaseSensitive() { assertThatThrownBy(() -> newSessionCatalog("TESTCONFIGURECATALOGCASESENSITIVE")) .isInstanceOf(RESTException.class) .hasMessage( @@ -397,7 +312,7 @@ public void testIcebergCreateNamespace() throws IOException { @Test public void testIcebergCreateNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -416,7 +331,7 @@ public void testIcebergCreateNamespaceInExternalCatalog(TestInfo testInfo) throw @Test public void testIcebergDropNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -433,7 +348,7 @@ public void testIcebergDropNamespaceInExternalCatalog(TestInfo testInfo) throws @Test public void testIcebergCreateTablesInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -460,56 +375,51 @@ public void testIcebergCreateTablesInExternalCatalog(TestInfo testInfo) throws I @Test public void testIcebergCreateTablesWithWritePathBlocked(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "Internal"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "Internal"; createCatalog(catalogName, Catalog.TypeEnum.INTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); - try { - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties( - Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - } catch (BadRequestException e) { - LOGGER.info("Received expected exception {}", e.getMessage()); - } + assertThatThrownBy( + () -> + sessionCatalog + .buildTable( + sessionContext, + TableIdentifier.of(ns, "the_table"), + new Schema( + List.of( + Types.NestedField.of( + 1, false, "theField", Types.StringType.get())))) + .withSortOrder(SortOrder.unsorted()) + .withPartitionSpec(PartitionSpec.unpartitioned()) + .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) + .create()) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Forbidden: Invalid locations"); + + assertThatThrownBy( + () -> + sessionCatalog + .buildTable( + sessionContext, + TableIdentifier.of(ns, "the_table"), + new Schema( + List.of( + Types.NestedField.of( + 1, false, "theField", Types.StringType.get())))) + .withSortOrder(SortOrder.unsorted()) + .withPartitionSpec(PartitionSpec.unpartitioned()) + .withProperties(Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) + .create()) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Forbidden: Invalid locations"); } } @Test public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, @@ -519,7 +429,7 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -556,7 +466,7 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws @Test public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, @@ -566,7 +476,7 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -609,7 +519,7 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO @Test public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, @@ -619,7 +529,7 @@ public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOEx .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -663,14 +573,14 @@ public void testWarehouseNotSpecified() throws IOException { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + apiClient.catalogApiEndpoint().toString(), OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", emptyEnvironmentVariable, - "header." + REALM_PROPERTY_KEY, + "header." + REALM_HEADER, realm))) .isInstanceOf(BadRequestException.class) .hasMessage("Malformed request: Please specify a warehouse"); @@ -679,12 +589,7 @@ public void testWarehouseNotSpecified() throws IOException { @Test public void testRequestHeaderTooLarge() { - Invocation.Builder request = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json"); + Invocation.Builder request = managementApi.request("v1/principal-roles"); // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely // exceeds the limit @@ -693,11 +598,7 @@ public void testRequestHeaderTooLarge() { } try { - try (Response response = - request - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(new PrincipalRole("r")))) { + try (Response response = request.post(Entity.json(new PrincipalRole("r")))) { assertThat(response) .returns( Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), @@ -715,36 +616,22 @@ public void testRequestBodyTooLarge() { // The size is set to be higher than the limit in polaris-server-integrationtest.yml Entity largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001))); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(largeRequest)) { + try (Response response = managementApi.request("v1/principal-roles").post(largeRequest)) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) - .matches( - r -> - r.readEntity(RequestThrottlingErrorResponse.class) - .errorType() - .equals(REQUEST_TOO_LARGE)); + .extracting(r -> r.readEntity(Map.class)) + .extracting("error_type") + .isEqualTo("REQUEST_TOO_LARGE"); } } @Test public void testRefreshToken() throws IOException { - String path = - String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", EXT.getLocalPort()); + String path = apiClient.catalogApiEndpoint().resolve("v1/oauth/tokens").toString(); try (RESTClient client = - HTTPClient.builder(ImmutableMap.of()) - .withHeader(REALM_PROPERTY_KEY, realm) - .uri(path) - .build()) { + HTTPClient.builder(Map.of()).withHeader(REALM_HEADER, realm).uri(path).build()) { String credentialString = - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(); + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(); var authConfig = AuthConfig.builder().credential(credentialString).scope("PRINCIPAL_ROLE:ALL").build(); ImmutableAuthConfig configSpy = spy(authConfig); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java similarity index 62% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java index b55b951fb..230cf1673 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java @@ -16,25 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.it.test; -import static io.dropwizard.jackson.Jackson.newObjectMapper; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest.PRINCIPAL_ROLE_ALL; import static org.assertj.core.api.Assertions.assertThat; import com.auth0.jwt.JWT; import com.auth0.jwt.JWTCreator; import com.auth0.jwt.algorithms.Algorithm; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; import java.io.IOException; import java.time.Duration; @@ -45,12 +39,7 @@ import java.util.Map; import java.util.UUID; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.rest.RESTUtil; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.responses.ErrorResponse; -import org.apache.iceberg.rest.responses.ListNamespacesResponse; -import org.apache.polaris.core.admin.model.AddGrantRequest; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; @@ -65,10 +54,8 @@ import org.apache.polaris.core.admin.model.CreatePrincipalRequest; import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; import org.apache.polaris.core.admin.model.ExternalCatalog; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; @@ -77,7 +64,6 @@ import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalRoles; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.Principals; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; @@ -85,28 +71,21 @@ import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.slf4j.LoggerFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) -public class PolarisServiceImplIntegrationTest { +@ExtendWith(PolarisIntegrationTestExtension.class) +public class PolarisManagementServiceIntegrationTest { private static final int MAX_IDENTIFIER_LENGTH = 256; private static final String ISSUER_KEY = "polaris"; private static final String CLAIM_KEY_ACTIVE = "active"; @@ -114,151 +93,52 @@ public class PolarisServiceImplIntegrationTest { private static final String CLAIM_KEY_PRINCIPAL_ID = "principalId"; private static final String CLAIM_KEY_SCOPE = "scope"; - // TODO: Add a test-only hook that fully clobbers all persistence state so we can have a fresh - // slate on every test case; otherwise, leftover state from one test from failures will interfere - // with other test cases. - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - - // disallow FILE urls for the sake of tests below - ConfigOverride.config( - "featureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES", "S3,GCS,AZURE"), - ConfigOverride.config("gcp_credentials.access_token", "abc"), - ConfigOverride.config("gcp_credentials.expires_in", "12345")); - private static String userToken; - private static String realm; - private static String clientId; + private static PolarisApiClient apiClient; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; + private static ClientCredentials rootCredentials; @BeforeAll public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, - PolarisPrincipalSecrets adminSecrets, - @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - clientId = adminSecrets.getPrincipalClientId(); - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + PolarisApiClient client, AuthToken adminToken, ClientCredentials credentials) { + apiClient = client; + managementApi = client.managementApi(adminToken); + catalogApi = client.catalogApi(adminToken); + rootCredentials = credentials; } @AfterEach public void tearDown() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - response - .readEntity(Catalogs.class) - .getCatalogs() - .forEach( - catalog -> { - // clean up the catalog before we try to drop it - - // delete all the namespaces - try (Response res = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list namespaces in catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - } else { - res.readEntity(ListNamespacesResponse.class) - .namespaces() - .forEach( - namespace -> { - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces/" - + RESTUtil.encodeNamespace(namespace)) - .delete() - .close(); - }); - } - } - - // delete all the catalog roles except catalog_admin - try (Response res = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list catalog roles for catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - return; - } - res.readEntity(CatalogRoles.class).getRoles().stream() - .filter(cr -> !cr.getName().equals("catalog_admin")) - .forEach( - cr -> - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles/" - + cr.getName()) - .delete() - .close()); - } - - Response deleteResponse = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalog.getName()) - .delete(); - if (deleteResponse.getStatus() != Response.Status.NO_CONTENT.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to delete catalog {}: {}", - catalog.getName(), - deleteResponse.readEntity(String.class)); - } - deleteResponse.close(); - }); - } - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { - response.readEntity(Principals.class).getPrincipals().stream() - .filter( - principal -> - !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) - .forEach( - principal -> { - newRequest( - "http://localhost:%d/api/management/v1/principals/" + principal.getName()) - .delete() - .close(); - }); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { - response.readEntity(PrincipalRoles.class).getRoles().stream() - .filter( - principalRole -> - !principalRole - .getName() - .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) - .forEach( - principalRole -> { - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRole.getName()) - .delete() - .close(); - }); - } + managementApi + .listCatalogs() + .forEach( + catalog -> { + // clean up the catalog before we try to drop it + // delete all the namespaces + catalogApi + .listNamespaces(catalog.getName()) + .forEach(namespace -> catalogApi.deleteNamespaces(catalog.getName(), namespace)); + + // delete all the catalog roles except catalog_admin + managementApi.listCatalogRoles(catalog.getName()).stream() + .filter(cr -> !cr.getName().equals("catalog_admin")) + .forEach(role -> managementApi.deleteCatalogRole(catalog.getName(), role)); + + managementApi.deleteCatalog(catalog.getName()); + }); + + managementApi.listPrincipals().stream() + .filter( + principal -> !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) + .forEach(principal -> managementApi.deletePrincipal(principal)); + + managementApi.listPrincipalRoles().stream() + .filter( + principalRole -> + !principalRole + .getName() + .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) + .forEach(principalRole -> managementApi.deletePrincipalRole(principalRole)); } @Test @@ -289,7 +169,7 @@ public void testCatalogSerializing() throws IOException { @Test public void testListCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -304,23 +184,9 @@ public void testListCatalogs() { @Test public void testListCatalogsUnauthorized() { - Principal principal = new Principal("a_new_user"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", newToken).get()) { + PrincipalWithCredentials principal = managementApi.createPrincipal("a_new_user"); + AuthToken userToken = apiClient.obtainToken(principal); + try (Response response = apiClient.managementApi(userToken).request("v1/catalogs").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -328,7 +194,8 @@ public void testListCatalogsUnauthorized() { @Test public void testCreateCatalog() { try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + managementApi + .request("v1/catalogs") .post( Entity.json( "{\"catalog\":{\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"},\"storageConfigInfo\":{\"storageType\":\"S3\",\"roleArn\":\"arn:aws:iam::123456789012:role/my-role\",\"externalId\":\"externalId\",\"userArn\":\"userArn\",\"allowedLocations\":[\"s3://my-old-bucket/path/to/data\"]}}}"))) { @@ -336,8 +203,7 @@ public void testCreateCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -355,8 +221,6 @@ public void testCreateCatalogWithInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - ObjectMapper mapper = newObjectMapper(); - Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -364,12 +228,8 @@ public void testCreateCatalogWithInvalidName() { .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); } String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); @@ -391,16 +251,12 @@ public void testCreateCatalogWithInvalidName() { .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); assertThat(errorResponse.message()).contains("Invalid value:"); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); } } } @@ -424,12 +280,10 @@ public void testCreateCatalogWithAzureStorageConfig() { .setStorageConfigInfo(azureConfigInfo) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -457,12 +311,10 @@ public void testCreateCatalogWithGcpStorageConfig() { .setStorageConfigInfo(gcpConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -491,9 +343,7 @@ public void testCreateCatalogWithNullBaseLocation() { catalogNode.set("properties", mapper.createObjectNode()); ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } @@ -518,9 +368,7 @@ public void testCreateCatalogWithoutProperties() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -533,12 +381,11 @@ public void testCreateCatalogWithoutProperties() { } @Test - public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingException { + public void testCreateCatalogWithoutStorageConfig() { String catalogString = "{\"catalog\": {\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"}}}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(catalogString))) { + managementApi.request("v1/catalogs").post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -551,11 +398,10 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti } @Test - public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException { + public void testCreateCatalogWithUnparsableJson() { String catalogString = "{\"catalog\": {{\"bad data}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(catalogString))) { + managementApi.request("v1/catalogs").post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -568,33 +414,7 @@ public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException } @Test - public void testCreateCatalogWithDisallowedStorageConfig() throws JsonProcessingException { - FileStorageConfigInfo fileStorage = - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://")) - .build(); - String catalogName = "my-external-catalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(new CatalogProperties("file:///tmp/path/to/data")) - .setStorageConfigInfo(fileStorage) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - assertThat(error) - .isNotNull() - .returns("Unsupported storage type: FILE", ErrorResponse::message); - } - } - - @Test - public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonProcessingException { + public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() { AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -612,16 +432,13 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -638,10 +455,9 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro fetchedCatalog.getEntityVersion(), Map.of("foo", "bar"), null /* storageConfigIno */); // Successfully update - Catalog updatedCatalog = null; + Catalog updatedCatalog; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/" + catalogName).put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); updatedCatalog = response.readEntity(Catalog.class); @@ -653,66 +469,6 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro } } - @Test - public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessingException { - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::123456789012:role/my-role") - .setExternalId("externalId") - .setUserArn("userArn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - String catalogName = "mycatalog"; - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(new CatalogProperties("s3://bucket/path/to/data")) - .setStorageConfigInfo(awsConfigModel) - .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - fetchedCatalog = response.readEntity(Catalog.class); - - assertThat(fetchedCatalog.getName()).isEqualTo(catalogName); - assertThat(fetchedCatalog.getProperties().toMap()) - .isEqualTo(Map.of("default-base-location", "s3://bucket/path/to/data")); - assertThat(fetchedCatalog.getEntityVersion()).isGreaterThan(0); - } - - FileStorageConfigInfo fileStorage = - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://")) - .build(); - UpdateCatalogRequest updateRequest = - new UpdateCatalogRequest( - fetchedCatalog.getEntityVersion(), - Map.of("default-base-location", "file:///tmp/path/to/data/"), - fileStorage); - - // failure to update - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .put(Entity.json(updateRequest))) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); - ErrorResponse error = response.readEntity(ErrorResponse.class); - - assertThat(error).returns("Unsupported storage type: FILE", ErrorResponse::message); - } - } - @Test public void testCreateExternalCatalog() { AwsStorageConfigInfo awsConfigModel = @@ -733,10 +489,9 @@ public void testCreateExternalCatalog() { .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) .setStorageConfigInfo(awsConfigModel) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).get()) { + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog fetchedCatalog = response.readEntity(Catalog.class); assertThat(fetchedCatalog) @@ -752,8 +507,7 @@ public void testCreateExternalCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).delete()) { + try (Response response = managementApi.request("v1/catalogs/" + catalogName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -780,16 +534,14 @@ public void testCreateCatalogWithoutDefaultLocation() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } } @Test - public void serialization() throws JsonProcessingException { + public void serialization() { CatalogProperties properties = new CatalogProperties("s3://my-bucket/path/to/data"); ObjectMapper mapper = new ObjectMapper(); CatalogProperties translated = mapper.convertValue(properties, CatalogProperties.class); @@ -810,12 +562,11 @@ public void testCreateAndUpdateAzureCatalog() { .build(); // 200 Successful create - createCatalog(catalog); + managementApi.createCatalog(catalog); // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/myazurecatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -834,8 +585,7 @@ public void testCreateAndUpdateAzureCatalog() { Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"), modifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(badUpdateRequest))) { + managementApi.request("v1/catalogs/myazurecatalog").put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -854,8 +604,7 @@ public void testCreateAndUpdateAzureCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/myazurecatalog").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -865,8 +614,7 @@ public void testCreateAndUpdateAzureCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/myazurecatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -884,19 +632,17 @@ public void testCreateListUpdateAndDeleteCatalog() { .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -907,7 +653,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // Should list the catalog. - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -927,8 +673,7 @@ public void testCreateListUpdateAndDeleteCatalog() { Map.of("default-base-location", "s3://newbucket/"), invalidModifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(badUpdateRequest))) { + managementApi.request("v1/catalogs/mycatalog").put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -953,8 +698,7 @@ public void testCreateListUpdateAndDeleteCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/mycatalog").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -966,8 +710,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -976,19 +719,17 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -1001,23 +742,10 @@ public void testCreateListUpdateAndDeleteCatalog() { } } - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); - } - - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); - } - @Test public void testGetCatalogNotFound() { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } } @@ -1035,9 +763,7 @@ public void testGetCatalogInvalidName() { for (String invalidCatalogName : invalidCatalogNames) { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + invalidCatalogName) - .get()) { + try (Response response = managementApi.request("v1/catalogs/" + invalidCatalogName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1058,7 +784,7 @@ public void testCatalogRoleInvalidName() { new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); List invalidCatalogRoleNames = @@ -1071,9 +797,8 @@ public void testCatalogRoleInvalidName() { for (String invalidCatalogRoleName : invalidCatalogRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/" - + invalidCatalogRoleName) + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles/" + invalidCatalogRoleName) .get()) { assertThat(response) @@ -1087,23 +812,9 @@ public void testCatalogRoleInvalidName() { @Test public void testListPrincipalsUnauthorized() { - Principal principal = new Principal("new_admin"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + PrincipalWithCredentials principal = managementApi.createPrincipal("new_admin"); + AuthToken userToken = apiClient.obtainToken(principal); + try (Response response = apiClient.managementApi(userToken).request("v1/principals").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -1116,40 +827,29 @@ public void testCreatePrincipalAndRotateCredentials() { .setProperties(Map.of("custom-tag", "foo")) .build(); - PrincipalWithCredentialsCredentials creds = null; - Principal returnedPrincipal = null; + PrincipalWithCredentials creds; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, true)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); + creds = response.readEntity(PrincipalWithCredentials.class); } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); - - String oldClientId = creds.getClientId(); - String oldSecret = creds.getClientSecret(); + assertThat(creds.getCredentials().getClientId()).isEqualTo(creds.getPrincipal().getClientId()); // Now rotate the credentials. First, if we try to just use the adminToken to rotate the // newly created principal's credentials, we should fail; rotateCredentials is only // a "self" privilege that even admins can't inherit. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal/rotate") - .post(Entity.json(""))) { + managementApi.request("v1/principals/myprincipal/rotate").post(Entity.json(""))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } - // Get a fresh token associate with the principal itself. - String newPrincipalToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), EXT.getLocalPort(), oldClientId, oldSecret, realm); + AuthToken oldUserToken = apiClient.obtainToken(creds); // Any call should initially fail with error indicating that rotation is needed. try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal", newPrincipalToken) - .get()) { + apiClient.managementApi(oldUserToken).request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); assertThat(error) @@ -1160,21 +860,23 @@ public void testCreatePrincipalAndRotateCredentials() { } // Now try to rotate using the principal's token. + PrincipalWithCredentials newCreds; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal/rotate", - newPrincipalToken) + apiClient + .managementApi(oldUserToken) + .request("v1/principals/myprincipal/rotate") .post(Entity.json(""))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); + newCreds = response.readEntity(PrincipalWithCredentials.class); } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); + assertThat(newCreds.getCredentials().getClientId()) + .isEqualTo(newCreds.getPrincipal().getClientId()); // ClientId shouldn't change - assertThat(creds.getClientId()).isEqualTo(oldClientId); - assertThat(creds.getClientSecret()).isNotEqualTo(oldSecret); + assertThat(newCreds.getCredentials().getClientId()) + .isEqualTo(creds.getCredentials().getClientId()); + assertThat(newCreds.getCredentials().getClientSecret()) + .isNotEqualTo(creds.getCredentials().getClientSecret()); // TODO: Test the validity of the old secret for getting tokens, here and then after a second // rotation that makes the old secret fall off retention. @@ -1188,22 +890,23 @@ public void testCreateListUpdateAndDeletePrincipal() { .setProperties(Map.of("custom-tag", "foo")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Principal fetchedPrincipal = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + Principal fetchedPrincipal; + try (Response response = managementApi.request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1213,7 +916,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // Should list the principal. - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = managementApi.request("v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1228,8 +931,7 @@ public void testCreateListUpdateAndDeletePrincipal() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal") - .put(Entity.json(updateRequest))) { + managementApi.request("v1/principals/myprincipal").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1237,8 +939,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = managementApi.request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1246,19 +947,17 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").delete()) { + try (Response response = managementApi.request("v1/principals/myprincipal").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = managementApi.request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = managementApi.request("v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1277,7 +976,8 @@ public void testCreatePrincipalWithInvalidName() { .setProperties(Map.of("custom-tag", "good_principal")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -1300,7 +1000,8 @@ public void testCreatePrincipalWithInvalidName() { .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1324,8 +1025,7 @@ public void testGetPrincipalWithInvalidName() { for (String invalidPrincipalName : invalidPrincipalNames) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/" + invalidPrincipalName) - .get()) { + managementApi.request("v1/principals/" + invalidPrincipalName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1339,20 +1039,20 @@ public void testGetPrincipalWithInvalidName() { public void testCreateListUpdateAndDeletePrincipalRole() { PrincipalRole principalRole = new PrincipalRole("myprincipalrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); - createPrincipalRole(principalRole); + managementApi.createPrincipalRole(principalRole); // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + managementApi + .request("v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - PrincipalRole fetchedPrincipalRole = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + PrincipalRole fetchedPrincipalRole; + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1363,8 +1063,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // Should list the principalRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = managementApi.request("v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1380,7 +1079,8 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") + managementApi + .request("v1/principal-roles/myprincipalrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1389,8 +1089,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1398,23 +1097,19 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = managementApi.request("v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1430,7 +1125,7 @@ public void testCreatePrincipalRoleInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); PrincipalRole principalRole = new PrincipalRole(goodName, Map.of("custom-tag", "good_principal_role"), 0L, 0L, 1); - createPrincipalRole(principalRole); + managementApi.createPrincipalRole(principalRole); String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); List invalidPrincipalRoleNames = @@ -1448,7 +1143,8 @@ public void testCreatePrincipalRoleInvalidName() { invalidPrincipalRoleName, Map.of("custom-tag", "bad_principal_role"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + managementApi + .request("v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1472,10 +1168,7 @@ public void testGetPrincipalRoleInvalidName() { for (String invalidPrincipalRoleName : invalidPrincipalRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + invalidPrincipalRoleName) - .get()) { + managementApi.request("v1/principal-roles/" + invalidPrincipalRoleName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1496,7 +1189,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); Catalog catalog2 = PolarisCatalog.builder() @@ -1507,12 +1200,13 @@ public void testCreateListUpdateAndDeleteCatalogRole() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://required/base/other_location")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); CatalogRole catalogRole = new CatalogRole("mycatalogrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1520,18 +1214,17 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - CatalogRole fetchedCatalogRole = null; + CatalogRole fetchedCatalogRole; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1542,9 +1235,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // Should list the catalogRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1555,9 +1246,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // Empty list if listing in catalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2/catalog-roles") - .get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog2/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1578,8 +1267,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 successful update try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1589,9 +1278,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 GET after update should show new properties try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1600,26 +1287,20 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 204 Successful delete try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .delete()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1630,15 +1311,13 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1649,7 +1328,8 @@ public void testAssignListAndRevokePrincipalRoles() { // Create two Principals Principal principal1 = new Principal("myprincipal1"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal1, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1657,7 +1337,8 @@ public void testAssignListAndRevokePrincipalRoles() { Principal principal2 = new Principal("myprincipal2"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal2, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1665,11 +1346,12 @@ public void testAssignListAndRevokePrincipalRoles() { // One PrincipalRole PrincipalRole principalRole = new PrincipalRole("myprincipalrole"); - createPrincipalRole(principalRole); + managementApi.createPrincipalRole(principalRole); // Assign the role to myprincipal1 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") + managementApi + .request("v1/principals/myprincipal1/principal-roles") .put(Entity.json(principalRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1677,8 +1359,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + managementApi.request("v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1692,9 +1373,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipal1 if listing assignees of myprincipalrole try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + managementApi.request("v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1707,8 +1386,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list if listing in principal2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2/principal-roles") - .get()) { + managementApi.request("v1/principals/myprincipal2/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1718,8 +1396,8 @@ public void testAssignListAndRevokePrincipalRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") + managementApi + .request("v1/principals/myprincipal1/principal-roles/myprincipalrole") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1727,8 +1405,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + managementApi.request("v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1736,9 +1413,7 @@ public void testAssignListAndRevokePrincipalRoles() { .returns(List.of(), PrincipalRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + managementApi.request("v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1747,23 +1422,19 @@ public void testAssignListAndRevokePrincipalRoles() { } // 204 Successful delete myprincipal1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1").delete()) { + try (Response response = managementApi.request("v1/principals/myprincipal1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipal2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2").delete()) { + try (Response response = managementApi.request("v1/principals/myprincipal2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipalrole - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1773,10 +1444,10 @@ public void testAssignListAndRevokePrincipalRoles() { public void testAssignListAndRevokeCatalogRoles() { // Create two PrincipalRoles PrincipalRole principalRole1 = new PrincipalRole("mypr1"); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); PrincipalRole principalRole2 = new PrincipalRole("mypr2"); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // One CatalogRole Catalog catalog = @@ -1788,11 +1459,12 @@ public void testAssignListAndRevokeCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); CatalogRole catalogRole = new CatalogRole("mycr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles") + managementApi + .request("v1/catalogs/mycatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1808,11 +1480,12 @@ public void testAssignListAndRevokeCatalogRoles() { new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(otherCatalog); + managementApi.createCatalog(otherCatalog); CatalogRole otherCatalogRole = new CatalogRole("myothercr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles") + managementApi + .request("v1/catalogs/othercatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1820,15 +1493,15 @@ public void testAssignListAndRevokeCatalogRoles() { // Assign both the roles to mypr1 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") + managementApi + .request("v1/principal-roles/mypr1/catalog-roles/mycatalog") .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") + managementApi + .request("v1/principal-roles/mypr1/catalog-roles/othercatalog") .put(Entity.json(new GrantCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1836,9 +1509,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list only mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + managementApi.request("v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1851,9 +1522,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list mypr1 if listing assignees of mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") - .get()) { + managementApi.request("v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1866,9 +1535,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Empty list if listing in principalRole2 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog") - .get()) { + managementApi.request("v1/principal-roles/mypr2/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1878,18 +1545,14 @@ public void testAssignListAndRevokeCatalogRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") - .delete()) { + managementApi.request("v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // Empty list try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + managementApi.request("v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1897,9 +1560,7 @@ public void testAssignListAndRevokeCatalogRoles() { .returns(List.of(), CatalogRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") - .get()) { + managementApi.request("v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1908,46 +1569,39 @@ public void testAssignListAndRevokeCatalogRoles() { } // 204 Successful delete mypr1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr1").delete()) { + try (Response response = managementApi.request("v1/principal-roles/mypr1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mypr2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr2").delete()) { + try (Response response = managementApi.request("v1/principal-roles/mypr2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycr try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr") - .delete()) { + managementApi.request("v1/catalogs/mycatalog/catalog-roles/mycr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myothercr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr") - .delete()) { + managementApi.request("v1/catalogs/othercatalog/catalog-roles/myothercr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete othercatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/othercatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1958,8 +1612,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal // role String principalRoleName = "mypr33"; - PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRoleName); String catalogName = "myuniquetestcatalog"; Catalog catalog = @@ -1971,56 +1624,54 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); + PrincipalWithCredentials catalogAdminPrincipal = managementApi.createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRoleName); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + AuthToken catalogAdminToken = apiClient.obtainToken(catalogAdminPrincipal); // Create a second principal role. Use the catalog admin principal to list principal roles and // grant a catalog role to the new principal role String principalRoleName2 = "mypr2"; PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // create a catalog role and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName, catalogRoleName, catalogAdminToken); + apiClient.managementApi(catalogAdminToken).createCatalogRole(catalogName, catalogRoleName); CatalogPrivilege privilege = CatalogPrivilege.CATALOG_MANAGE_CONTENT; - grantPrivilegeToCatalogRole( - catalogName, - catalogRoleName, - new CatalogGrant(privilege, GrantResource.TypeEnum.CATALOG), - catalogAdminToken, - Response.Status.CREATED); + apiClient + .managementApi(catalogAdminToken) + .addGrant( + catalogName, + catalogRoleName, + new CatalogGrant(privilege, GrantResource.TypeEnum.CATALOG)); // The catalog admin can grant the new catalog role to the mypr2 principal role - grantCatalogRoleToPrincipalRole( - principalRoleName2, catalogName, new CatalogRole(catalogRoleName), catalogAdminToken); + apiClient + .managementApi(catalogAdminToken) + .grantCatalogRoleToPrincipalRole( + principalRoleName2, catalogName, new CatalogRole(catalogRoleName)); // But the catalog admin cannot revoke the role because it requires // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + apiClient + .managementApi(catalogAdminToken) + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" - + catalogRoleName, - catalogAdminToken) + + catalogRoleName) .delete()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } @@ -2028,14 +1679,14 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // The service admin can revoke the role because it has the // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE privilege try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + managementApi + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" - + catalogRoleName, - userToken) + + catalogRoleName) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -2047,7 +1698,7 @@ public void testServiceAdminCanTransferCatalogAdmin() { // role String principalRoleName = "mypr33"; PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); String catalogName = "myothertestcatalog"; Catalog catalog = @@ -2059,31 +1710,26 @@ public void testServiceAdminCanTransferCatalogAdmin() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); + PrincipalWithCredentials catalogAdminPrincipal = managementApi.createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + AuthToken catalogAdminToken = apiClient.obtainToken(catalogAdminPrincipal); // service_admin revokes the catalog_admin privilege from its principal role try { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/service_admin/catalog-roles/" + managementApi + .request( + "v1/principal-roles/service_admin/catalog-roles/" + catalogName - + "/catalog_admin", - userToken) + + "/catalog_admin") .delete()) { assertThat(response) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -2091,21 +1737,23 @@ public void testServiceAdminCanTransferCatalogAdmin() { // the service_admin can not revoke the catalog_admin privilege from the new principal role try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + apiClient + .managementApi(catalogAdminToken) + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName - + "/catalog_admin", - catalogAdminToken) + + "/catalog_admin") .delete()) { assertThat(response) .returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } finally { // grant the admin role back to service_admin so that cleanup can happen - grantCatalogRoleToPrincipalRole( - "service_admin", catalogName, catalogAdminRole, catalogAdminToken); + apiClient + .managementApi(catalogAdminToken) + .grantCatalogRoleToPrincipalRole("service_admin", catalogName, catalogAdminRole); } } @@ -2115,7 +1763,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // role String principalRoleName = "mypr33"; PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); // create a catalog String catalogName = "mytestcatalog"; @@ -2128,7 +1776,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a second catalog String catalogName2 = "anothercatalog"; @@ -2141,42 +1789,34 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); // create a catalog role *in the second catalog* and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName2, catalogRoleName, userToken); + managementApi.createCatalogRole(catalogName2, catalogRoleName); // Get the catalog admin role from the *first* catalog and grant that role to the principal role - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + PrincipalWithCredentials catalogAdminPrincipal = managementApi.createPrincipal("principal1"); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + AuthToken catalogAdminToken = apiClient.obtainToken(catalogAdminPrincipal); // Create a second principal role. String principalRoleName2 = "mypr2"; PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // The catalog admin cannot grant the new catalog role to the mypr2 principal role because the // catalog role is in the wrong catalog try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName2, - catalogAdminToken) + apiClient + .managementApi(catalogAdminToken) + .request("v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName2) .put(Entity.json(new GrantCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } @@ -2187,7 +1827,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // Create a PrincipalRole and a new catalog. String principalRoleName = "mypr33"; PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); // create a catalog String catalogName = "mytablemanagecatalog"; @@ -2200,10 +1840,10 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a valid target CatalogRole in this catalog - createCatalogRole(catalogName, "target_catalog_role", userToken); + managementApi.createCatalogRole(catalogName, "target_catalog_role"); // create a second catalog String catalogName2 = "anothertablemanagecatalog"; @@ -2216,59 +1856,53 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); // create an *invalid* target CatalogRole in second catalog - createCatalogRole(catalogName2, "invalid_target_catalog_role", userToken); + managementApi.createCatalogRole(catalogName2, "invalid_target_catalog_role"); // create the namespace "c" in *both* namespaces String namespaceName = "c"; - createNamespace(catalogName, namespaceName); - createNamespace(catalogName2, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName2, namespaceName); // create a catalog role *in the first catalog* and grant it manage_content privilege at the // namespace level // grant that role to the PrincipalRole String catalogRoleName = "ns_manage_access_role"; - createCatalogRole(catalogName, catalogRoleName, userToken); - grantPrivilegeToCatalogRole( + managementApi.createCatalogRole(catalogName, catalogRoleName); + managementApi.addGrant( catalogName, catalogRoleName, new NamespaceGrant( List.of(namespaceName), NamespacePrivilege.CATALOG_MANAGE_ACCESS, - GrantResource.TypeEnum.NAMESPACE), - userToken, - Response.Status.CREATED); + GrantResource.TypeEnum.NAMESPACE)); - grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, new CatalogRole(catalogRoleName), userToken); + managementApi.grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, new CatalogRole(catalogRoleName)); // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("ns_manage_access_user"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + PrincipalWithCredentials catalogAdminPrincipal = + managementApi.createPrincipal("ns_manage_access_user"); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String manageAccessUserToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + AuthToken manageAccessUserToken = apiClient.obtainToken(catalogAdminPrincipal); // Use the ns_manage_access_user to grant TABLE_CREATE access to the target catalog role // This works because the user has CATALOG_MANAGE_ACCESS within the namespace and the target // catalog role is in // the same catalog - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.CREATED); + apiClient + .managementApi(manageAccessUserToken) + .addGrant( + catalogName, + "target_catalog_role", + new NamespaceGrant( + List.of(namespaceName), + NamespacePrivilege.TABLE_CREATE, + GrantResource.TypeEnum.NAMESPACE)); // Even though the ns_manage_access_role can grant privileges to the catalog role, it cannot // grant the target @@ -2276,45 +1910,59 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // on the catalog role // as a securable try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - manageAccessUserToken) + apiClient + .managementApi(manageAccessUserToken) + .request("v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName) .put( Entity.json(new GrantCatalogRoleRequest(new CatalogRole("target_catalog_role"))))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } // The user cannot grant catalog-level privileges to the catalog role - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + apiClient + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", "target_catalog_role")) + .put( + Entity.json( + new CatalogGrant( + CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } // even though the namespace "c" exists in both catalogs, the ns_manage_access_role can only // grant privileges for // the namespace in its own catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + apiClient + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName2, "role", "invalid_target_catalog_role")) + .put( + Entity.json( + new NamespaceGrant( + List.of(namespaceName), + NamespacePrivilege.TABLE_CREATE, + GrantResource.TypeEnum.NAMESPACE)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } // nor can it grant privileges to the catalog role in the second catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + apiClient + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName2, "role", "invalid_target_catalog_role")) + .put( + Entity.json( + new CatalogGrant( + CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } } @Test @@ -2325,13 +1973,16 @@ public void testTokenExpiry() { .withExpiresAt(Instant.now().plus(1, ChronoUnit.SECONDS)) .sign(Algorithm.HMAC256("polaris")); Awaitility.await("expected list of records should be produced") - .atMost(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(20)) .pollDelay(Duration.ofSeconds(1)) .pollInterval(Duration.ofSeconds(1)) .untilAsserted( () -> { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + apiClient + .managementApi(new AuthToken(newToken, "test-user")) + .request("v1/principals") + .get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2344,7 +1995,10 @@ public void testTokenInactive() { String newToken = defaultJwt().withClaim(CLAIM_KEY_ACTIVE, false).sign(Algorithm.HMAC256("polaris")); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + apiClient + .managementApi(new AuthToken(newToken, "test-user")) + .request("v1/principals") + .get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2355,7 +2009,10 @@ public void testTokenInvalidSignature() { // SignatureVerificationException - if the signature is invalid. String newToken = defaultJwt().sign(Algorithm.HMAC256("invalid_secret")); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + apiClient + .managementApi(new AuthToken(newToken, "test-user")) + .request("v1/principals") + .get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2366,7 +2023,10 @@ public void testTokenInvalidPrincipalId() { String newToken = defaultJwt().withClaim(CLAIM_KEY_PRINCIPAL_ID, 0).sign(Algorithm.HMAC256("polaris")); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + apiClient + .managementApi(new AuthToken(newToken, "test-user")) + .request("v1/principals") + .get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2385,21 +2045,15 @@ public void testNamespaceExistsStatus() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a namespace String namespaceName = "c"; - createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); // check if a namespace existed try (Response response = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) - .head()) { + catalogApi.request("v1/" + catalogName + "/namespaces/" + namespaceName).head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -2417,21 +2071,15 @@ public void testDropNamespaceStatus() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a namespace String namespaceName = "c"; - createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); // drop a namespace try (Response response = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) - .delete()) { + catalogApi.request("v1/" + catalogName + "/namespaces/" + namespaceName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -2445,119 +2093,8 @@ public static JWTCreator.Builder defaultJwt() { .withExpiresAt(now.plus(10, ChronoUnit.SECONDS)) .withJWTId(UUID.randomUUID().toString()) .withClaim(CLAIM_KEY_ACTIVE, true) - .withClaim(CLAIM_KEY_CLIENT_ID, clientId) + .withClaim(CLAIM_KEY_CLIENT_ID, rootCredentials.clientId()) .withClaim(CLAIM_KEY_PRINCIPAL_ID, 1) - .withClaim(CLAIM_KEY_SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL); - } - - private static void createNamespace(String catalogName, String namespaceName) { - try (Response response = - newRequest("http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", userToken) - .post( - Entity.json( - CreateNamespaceRequest.builder() - .withNamespace(Namespace.of(namespaceName)) - .build()))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalog(Catalog catalog) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void grantPrivilegeToCatalogRole( - String catalogName, - String catalogRoleName, - GrantResource grant, - String catalogAdminToken, - Response.Status expectedStatus) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + catalogRoleName - + "/grants", - catalogAdminToken) - .put(Entity.json(new AddGrantRequest(grant)))) { - assertThat(response).returns(expectedStatus.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalogRole( - String catalogName, String catalogRoleName, String catalogAdminToken) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", - catalogAdminToken) - .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void grantPrincipalRoleToPrincipal( - String principalName, PrincipalRole principalRole) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/" - + principalName - + "/principal-roles") - .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static PrincipalWithCredentials createPrincipal(String principalName) { - PrincipalWithCredentials catalogAdminPrincipal; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(new Principal(principalName), false)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - catalogAdminPrincipal = response.readEntity(PrincipalWithCredentials.class); - } - return catalogAdminPrincipal; - } - - private static void grantCatalogRoleToPrincipalRole( - String principalRoleName, String catalogName, CatalogRole catalogRole, String token) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - token) - .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static CatalogRole readCatalogRole(String catalogName, String roleName) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + roleName) - .get()) { - - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - return response.readEntity(CatalogRole.class); - } - } - - private static void createPrincipalRole(PrincipalRole principalRole1) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") - .post(Entity.json(new CreatePrincipalRoleRequest(principalRole1)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + .withClaim(CLAIM_KEY_SCOPE, PRINCIPAL_ROLE_ALL); } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java similarity index 62% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 948f6d581..2d3d2fc09 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -16,22 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,33 +60,27 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogPrivilege; -import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.GrantResources; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.TableGrant; import org.apache.polaris.core.admin.model.TablePrivilege; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; import org.apache.polaris.core.admin.model.ViewGrant; import org.apache.polaris.core.admin.model.ViewPrivilege; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.BeforeAll; @@ -103,12 +93,7 @@ * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisRestCatalogIntegrationTest extends CatalogTests { private static final String TEST_ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) @@ -116,22 +101,16 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static final String S3_BUCKET_BASE = Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) .orElse("file:///tmp/buckets/my-bucket"); - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; + private static String principalRoleName; + private static PrincipalWithCredentials principalCredentials; + private static PolarisApiClient apiClient; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; private RESTCatalog restCatalog; private String currentCatalogName; - private String userToken; - private String realm; private final String catalogBaseLocation = S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; @@ -157,93 +136,73 @@ String[] properties() default { } @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + static void setup(PolarisApiClient client, AuthToken adminToken) { + apiClient = client; + managementApi = client.managementApi(adminToken); + String principalName = "snowman-rest-" + UUID.randomUUID(); + principalRoleName = "rest-admin-" + UUID.randomUUID(); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + catalogApi = client.catalogApi(principalCredentials); } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) { - this.realm = realm; - userToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - snowmanCredentials.clientId(), - snowmanCredentials.clientSecret(), - realm); - testInfo - .getTestMethod() - .ifPresent( - method -> { - currentCatalogName = method.getName() + UUID.randomUUID(); - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn(TEST_ROLE_ARN) - .setExternalId("externalId") - .setUserArn("a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - Optional catalogConfig = - testInfo - .getTestMethod() - .flatMap(m -> Optional.ofNullable(m.getAnnotation(CatalogConfig.class))); - - org.apache.polaris.core.admin.model.CatalogProperties.Builder catalogPropsBuilder = - org.apache.polaris.core.admin.model.CatalogProperties.builder( - catalogBaseLocation); - String[] properties = - catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); - for (int i = 0; i < properties.length; i += 2) { - catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); - } - if (!S3_BUCKET_BASE.startsWith("file:/")) { - catalogPropsBuilder.addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); - } - Catalog catalog = - PolarisCatalog.builder() - .setType( - catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) - .setName(currentCatalogName) - .setProperties(catalogPropsBuilder.build()) - .setStorageConfigInfo( - S3_BUCKET_BASE.startsWith("file:/") - ? new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) - : awsConfigModel) - .build(); - - Optional restCatalogConfig = - testInfo - .getTestMethod() - .flatMap( - m -> - Optional.ofNullable( - m.getAnnotation( - PolarisRestCatalogIntegrationTest.RestCatalogConfig.class))); - ImmutableMap.Builder extraPropertiesBuilder = - ImmutableMap.builder(); - restCatalogConfig.ifPresent( - config -> { - for (int i = 0; i < config.value().length; i += 2) { - extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); - } - }); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - EXT, - adminToken, - snowmanCredentials, - realm, - catalog, - extraPropertiesBuilder.build()); - }); + public void before(TestInfo testInfo) { + Method method = testInfo.getTestMethod().orElseThrow(); + currentCatalogName = method.getName() + UUID.randomUUID(); + AwsStorageConfigInfo awsConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn(TEST_ROLE_ARN) + .setExternalId("externalId") + .setUserArn("a:user:arn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) + .build(); + Optional catalogConfig = + Optional.ofNullable(method.getAnnotation(CatalogConfig.class)); + + CatalogProperties.Builder catalogPropsBuilder = CatalogProperties.builder(catalogBaseLocation); + String[] properties = + catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); + for (int i = 0; i < properties.length; i += 2) { + catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); + } + if (!S3_BUCKET_BASE.startsWith("file:/")) { + catalogPropsBuilder.addProperty( + CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); + } + Catalog catalog = + PolarisCatalog.builder() + .setType(catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) + .setName(currentCatalogName) + .setProperties(catalogPropsBuilder.build()) + .setStorageConfigInfo( + S3_BUCKET_BASE.startsWith("file:/") + ? new FileStorageConfigInfo( + StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) + : awsConfigModel) + .build(); + + managementApi.createCatalog(principalRoleName, catalog); + + Optional restCatalogConfig = + testInfo + .getTestMethod() + .flatMap( + m -> + Optional.ofNullable( + m.getAnnotation( + PolarisRestCatalogIntegrationTest.RestCatalogConfig.class))); + ImmutableMap.Builder extraPropertiesBuilder = ImmutableMap.builder(); + restCatalogConfig.ifPresent( + config -> { + for (int i = 0; i < config.value().length; i += 2) { + extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); + } + }); + + restCatalog = + IcebergHelper.restCatalog( + apiClient, principalCredentials, currentCatalogName, extraPropertiesBuilder.build()); } @Override @@ -271,37 +230,6 @@ protected boolean overridesRequestedLocation() { return true; } - private void createCatalogRole(String catalogRoleName) { - CatalogRole catalogRole = new CatalogRole(catalogRoleName); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalogRole))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private void addGrant(String catalogRoleName, GrantResource grant) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, catalogRoleName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .put(Entity.json(grant))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - @Test public void testListGrantsOnCatalogObjectsToCatalogRoles() { restCatalog.createNamespace(Namespace.of("ns1")); @@ -396,8 +324,8 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { ViewPrivilege.VIEW_WRITE_PROPERTIES, GrantResource.TypeEnum.VIEW); - createCatalogRole("catalogrole1"); - createCatalogRole("catalogrole2"); + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + managementApi.createCatalogRole(currentCatalogName, "catalogrole2"); List role1Grants = List.of( @@ -409,7 +337,7 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { tableGrant2, viewGrant1, viewGrant2); - role1Grants.forEach(grant -> addGrant("catalogrole1", grant)); + role1Grants.forEach(grant -> managementApi.addGrant(currentCatalogName, "catalogrole1", grant)); List role2Grants = List.of( catalogGrant1, @@ -420,45 +348,19 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { tableGrant3, viewGrant1, viewGrant3); - role2Grants.forEach(grant -> addGrant("catalogrole2", grant)); + role2Grants.forEach(grant -> managementApi.addGrant(currentCatalogName, "catalogrole2", grant)); // List grants for catalogrole1 - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role1Grants.toArray(new GrantResource[0])); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole1")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactlyInAnyOrder(role1Grants.toArray(new GrantResource[0])); // List grants for catalogrole2 - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole2")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role2Grants.toArray(new GrantResource[0])); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole2")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactlyInAnyOrder(role2Grants.toArray(new GrantResource[0])); } @Test @@ -478,8 +380,8 @@ public void testListGrantsAfterRename() { TablePrivilege.TABLE_FULL_METADATA, GrantResource.TypeEnum.TABLE); - createCatalogRole("catalogrole1"); - addGrant("catalogrole1", tableGrant1); + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + managementApi.addGrant(currentCatalogName, "catalogrole1", tableGrant1); // Grants will follow the table through the rename restCatalog.renameTable( @@ -493,60 +395,19 @@ public void testListGrantsAfterRename() { TablePrivilege.TABLE_FULL_METADATA, GrantResource.TypeEnum.TABLE); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactly(expectedGrant); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole1")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactly(expectedGrant); } @Test - public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocation() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -569,41 +430,12 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { } @Test - public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( - PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -635,41 +467,12 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( } @Test - public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( - PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -790,27 +593,20 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled @Test public void testSendNotificationInternalCatalog() { - NotificationRequest notification = new NotificationRequest(); - notification.setNotificationType(NotificationType.CREATE); - notification.setPayload( - new TableUpdateNotification( - "tbl1", - System.currentTimeMillis(), - UUID.randomUUID().toString(), - "s3://my-bucket/path/to/metadata.json", - null)); + Map payload = + ImmutableMap.builder() + .put("table-name", "tbl1") + .put("timestamps", "" + System.currentTimeMillis()) + .put("table-uuid", UUID.randomUUID().toString()) + .put("metadata-location", "s3://my-bucket/path/to/metadata.json") + .build(); restCatalog.createNamespace(Namespace.of("ns1")); - String notificationUrl = - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", - EXT.getLocalPort(), currentCatalogName); + Invocation.Builder notificationEndpoint = + catalogApi.request( + "v1/{cat}/namespaces/ns1/tables/tbl1/notifications", Map.of("cat", currentCatalogName)); try (Response response = - EXT.client() - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(notification))) { + notificationEndpoint.post( + Entity.json(Map.of("notification-type", "CREATE", "payload", payload)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(ErrorResponse.class)) @@ -818,14 +614,9 @@ public void testSendNotificationInternalCatalog() { } // NotificationType.VALIDATE should also surface the same error. - notification.setNotificationType(NotificationType.VALIDATE); try (Response response = - EXT.client() - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(notification))) { + notificationEndpoint.post( + Entity.json(Map.of("notification-type", "VALIDATE", "payload", payload)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(ErrorResponse.class)) @@ -1033,14 +824,10 @@ public void testTableExistsStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "table", tableName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1059,14 +846,10 @@ public void testDropTableStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "table", tableName)) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1093,14 +876,10 @@ public void testViewExistsStatus() { .create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1127,14 +906,10 @@ public void testDropViewStatus() { .create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1168,42 +943,28 @@ public void testRenameViewStatus() { // Perform view rename try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/views/rename", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request("v1/{cat}/views/rename", Map.of("cat", currentCatalogName)) .post(Entity.json(payload))) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // Original view should no longer exists try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .head()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } - // New view should exists + // New view should exist try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, newViewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", newViewName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java index b7fdbee57..d6b7e39cb 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.Optional; @@ -27,7 +27,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on AWS. */ public class PolarisRestCatalogViewAwsIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) // Backward compatibility .orElse(System.getenv("INTEGRATION_TEST_S3_ROLE_ARN")); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java index be755526f..4a4eef984 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.stream.Stream; @@ -26,7 +26,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on Azure. */ public class PolarisRestCatalogViewAzureIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String TENANT_ID = System.getenv("INTEGRATION_TEST_AZURE_TENANT_ID"); public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_AZURE_PATH"); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java similarity index 93% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java index c7853df97..6b6bc5548 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; @@ -24,7 +24,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on the local filesystem. */ public class PolarisRestCatalogViewFileIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String BASE_LOCATION = "file:///tmp/buckets/my-bucket"; @Override diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java index b74677275..f1c4a762a 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.stream.Stream; @@ -26,7 +26,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on GCP. */ public class PolarisRestCatalogViewGcpIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String SERVICE_ACCOUNT = System.getenv("INTEGRATION_TEST_GCS_SERVICE_ACCOUNT"); public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_GCS_PATH"); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java new file mode 100644 index 000000000..d1c010ed7 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import java.lang.reflect.Method; +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.view.ViewCatalogTests; +import org.apache.polaris.core.PolarisConfiguration; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog + * client. + */ +@ExtendWith(PolarisIntegrationTestExtension.class) +public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogTests { + + private static String principalRoleName; + private static PrincipalWithCredentials principalCredentials; + private static PolarisApiClient apiClient; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; + + private RESTCatalog restCatalog; + + @BeforeAll + static void setup(PolarisApiClient client, AuthToken adminToken) { + apiClient = client; + managementApi = client.managementApi(adminToken); + String principalName = "snowman-rest-" + UUID.randomUUID(); + principalRoleName = "rest-admin-" + UUID.randomUUID(); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + catalogApi = client.catalogApi(principalCredentials); + } + + @BeforeEach + public void before(TestInfo testInfo) { + + Assumptions.assumeFalse(shouldSkip()); + + Method method = testInfo.getTestMethod().orElseThrow(); + String catalogName = method.getName() + UUID.randomUUID(); + + StorageConfigInfo storageConfig = getStorageConfigInfo(); + String defaultBaseLocation = + storageConfig.getAllowedLocations().getFirst() + + "/" + + System.getenv("USER") + + "/path/to/data"; + + CatalogProperties props = + CatalogProperties.builder(defaultBaseLocation) + .addProperty( + CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:") + .addProperty(PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .build(); + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(props) + .setStorageConfigInfo(storageConfig) + .build(); + managementApi.createCatalog(principalRoleName, catalog); + + restCatalog = IcebergHelper.restCatalog(apiClient, principalCredentials, catalogName, Map.of()); + } + + /** + * @return The catalog's storage config. + */ + protected abstract StorageConfigInfo getStorageConfigInfo(); + + /** + * @return Whether the tests should be skipped, for example due to environment variables not being + * specified. + */ + protected abstract boolean shouldSkip(); + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected org.apache.iceberg.catalog.Catalog tableCatalog() { + return restCatalog; + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected boolean supportsServerSideRetry() { + return true; + } + + @Override + protected boolean overridesRequestedLocation() { + return true; + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java similarity index 67% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java index 173e2bda6..e8b04cf33 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java @@ -16,17 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.adobe.testing.s3mock.testcontainers.S3MockContainer; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import com.google.common.collect.ImmutableMap; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; @@ -41,14 +37,11 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; @@ -61,39 +54,22 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisSparkIntegrationTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism public static final String CATALOG_NAME = "mycatalog"; public static final String EXTERNAL_CATALOG_NAME = "external_catalog"; private static final S3MockContainer s3Container = new S3MockContainer("3.11.0").withInitialBuckets("my-bucket,my-old-bucket"); - private static PolarisConnectionExtension.PolarisToken polarisToken; private static SparkSession spark; - private String realm; + private PolarisApiClient apiClient; + private ManagementApi managementApi; + private CatalogApi catalogApi; + private AuthToken sparkToken; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken polarisToken, @PolarisRealm String realm) - throws IOException { + public static void setup() throws IOException { s3Container.start(); - PolarisSparkIntegrationTest.polarisToken = polarisToken; - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); } @AfterAll @@ -102,8 +78,12 @@ public static void cleanup() { } @BeforeEach - public void before(@PolarisRealm String realm) { - this.realm = realm; + public void before(PolarisApiClient client, AuthToken adminToken) { + apiClient = client; + sparkToken = adminToken; + managementApi = client.managementApi(adminToken); + catalogApi = client.catalogApi(adminToken); + AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -139,16 +119,7 @@ public void before(@PolarisRealm String realm) { .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + managementApi.createCatalog(catalog); CatalogProperties externalProps = new CatalogProperties("s3://my-bucket/path/to/data"); externalProps.putAll( @@ -177,16 +148,9 @@ public void before(@PolarisRealm String realm) { .setStorageConfigInfo(awsConfigModel) .setRemoteUrl("http://dummy_url") .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(externalCatalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + + managementApi.createCatalog(externalCatalog); + SparkSession.Builder sessionBuilder = SparkSession.builder() .master("local[1]") @@ -215,11 +179,11 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca .config(String.format("spark.sql.catalog.%s.type", catalogName), "rest") .config( String.format("spark.sql.catalog.%s.uri", catalogName), - "http://localhost:" + EXT.getLocalPort() + "/api/catalog") + apiClient.catalogApiEndpoint().toString()) .config(String.format("spark.sql.catalog.%s.warehouse", catalogName), catalogName) .config(String.format("spark.sql.catalog.%s.scope", catalogName), "PRINCIPAL_ROLE:ALL") - .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), realm) - .config(String.format("spark.sql.catalog.%s.token", catalogName), polarisToken.token()) + .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), apiClient.realm()) + .config(String.format("spark.sql.catalog.%s.token", catalogName), sparkToken.token()) .config(String.format("spark.sql.catalog.%s.s3.access-key-id", catalogName), "fakekey") .config( String.format("spark.sql.catalog.%s.s3.secret-access-key", catalogName), "fakesecret") @@ -253,18 +217,8 @@ private void cleanupCatalog(String catalogName) { } onSpark("DROP NAMESPACE " + namespace.getString(0)); } - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + + managementApi.deleteCatalog(catalogName); } @Test @@ -303,16 +257,9 @@ public void testCreateAndUpdateExternalTable() { LoadTableResponse tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); try (Response registerResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/" - + EXTERNAL_CATALOG_NAME - + "/namespaces/externalns1/register", - EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/externalns1/register", Map.of("cat", EXTERNAL_CATALOG_NAME)) .post( Entity.json( ImmutableRegisterTableRequest.builder() @@ -333,28 +280,28 @@ public void testCreateAndUpdateExternalTable() { onSpark("INSERT INTO " + CATALOG_NAME + ".ns1.tb1 VALUES (20, 'new_text')"); tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); - TableUpdateNotification updateNotification = - new TableUpdateNotification( - "mytb1", - Instant.now().toEpochMilli(), - tableResponse.tableMetadata().uuid(), - tableResponse.metadataLocation(), - tableResponse.tableMetadata()); - NotificationRequest notificationRequest = new NotificationRequest(); - notificationRequest.setPayload(updateNotification); - notificationRequest.setNotificationType(NotificationType.UPDATE); + Map updateNotification = + ImmutableMap.builder() + .put("table-name", "mytb1") + .put("timestamp", "" + Instant.now().toEpochMilli()) + .put("table-uuid", tableResponse.tableMetadata().uuid()) + .put("metadata-location", tableResponse.metadataLocation()) + .put("metadata", tableResponse.tableMetadata()) + .build(); + Map notificationRequest = + ImmutableMap.builder() + .put("payload", updateNotification) + .put("notification-type", "UPDATE") + .build(); try (Response notifyResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", - EXT.getLocalPort(), EXTERNAL_CATALOG_NAME)) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/externalns1/tables/mytb1/notifications", + Map.of("cat", EXTERNAL_CATALOG_NAME)) .post(Entity.json(notificationRequest))) { assertThat(notifyResponse) - .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); + .extracting(Response::getStatus) + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); } // refresh the table so it queries for the latest metadata.json onSpark("REFRESH TABLE mytb1"); @@ -378,14 +325,10 @@ public void testCreateView() { private LoadTableResponse loadTable(String catalog, String namespace, String table) { try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), catalog, namespace, table)) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", catalog, "ns", namespace, "table", table)) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); return response.readEntity(LoadTableResponse.class);