Add FileIOWrapper to allow intercepting File IO

Dockerfile

@@ -17,6 +17,7 @@
 # Use a non-docker-io registry, because pulling images from docker.io is
 # subject to aggressive request rate limiting and bandwidth shaping.
 FROM registry.access.redhat.com/ubi9/openjdk-21:1.20-2.1721752936 as build
+ARG ECLIPSELINK=false
 
 # Copy the REST catalog into the container
 COPY --chown=default:root . /app
@@ -26,14 +27,16 @@ WORKDIR /app
 RUN rm -rf build
 
 # Build the rest catalog
-RUN ./gradlew --no-daemon --info clean shadowJar
+RUN ./gradlew --no-daemon --info -PeclipseLink=$ECLIPSELINK clean shadowJar startScripts
 
 FROM registry.access.redhat.com/ubi9/openjdk-21-runtime:1.20-2.1721752928
 WORKDIR /app
-COPY --from=build /app/polaris-service/build/libs/polaris-service-1.0.0-all.jar /app
+COPY --from=build /app/polaris-service/build/libs/polaris-service-all.jar /app/lib/polaris-service-all.jar
 COPY --from=build /app/polaris-server.yml /app
+COPY --from=build /app/polaris-service/build/scripts/polaris-service /app/bin/polaris-service
 
 EXPOSE 8181
 
 # Run the resulting java binary
-CMD ["java", "-jar", "/app/polaris-service-1.0.0-all.jar", "server", "polaris-server.yml"]
+ENTRYPOINT ["/app/bin/polaris-service"]
+CMD ["server", "polaris-server.yml"]

README.md

@@ -34,6 +34,10 @@ Polaris Catalog is open source under an Apache 2.0 license.
 - ⭐ Star this repo if you’d like to bookmark and come back to it! 
 - 📖 Read the <a href="https://www.snowflake.com/blog/polaris-catalog-open-source/" target="_blank">announcement blog post<a/> for more details!
 
+## Development
+
+See [CONTRIBUTING](CONTRIBUTING.md) for contribution requirements.
+
 ## Building and Running 
 
 Polaris is organized into the following modules:

build.gradle.kts

@@ -86,8 +86,14 @@ tasks.named<RatTask>("rat").configure {
   excludes.add("polaris-service/src/**/banner.txt")
   excludes.add("polaris-service/logs")
 
+  excludes.add("**/polaris-venv/**")
+
+  excludes.add("**/.pytest_cache/**")
   excludes.add("regtests/**/py.typed")
   excludes.add("regtests/**/*.ref")
+  excludes.add("regtests/.env")
+  excludes.add("regtests/derby.log")
+  excludes.add("regtests/metastore_db/**")
   excludes.add("regtests/client/python/.openapi-generator/**")
 
   excludes.add("**/*.ipynb")

docker-compose.yml

@@ -27,14 +27,8 @@ services:
       AZURE_TENANT_ID: $AZURE_TENANT_ID
       AZURE_CLIENT_ID: $AZURE_CLIENT_ID
       AZURE_CLIENT_SECRET: $AZURE_CLIENT_SECRET
-    command: # override the command to specify aws keys as dropwizard config
-      - java
-      - -Ddw.awsAccessKey=$AWS_ACCESS_KEY_ID
-      - -Ddw.awsSecretKey=$AWS_SECRET_ACCESS_KEY
-      - -jar
-      - /app/polaris-service-1.0.0-all.jar
-      - server
-      - polaris-server.yml
+      # add aws keys as dropwizard config
+      JAVA_OPTS: -Ddw.awsAccessKey=$AWS_ACCESS_KEY_ID -Ddw.awsSecretKey=$AWS_SECRET_ACCESS_KEY
     volumes:
       - credentials:/tmp/credentials/
 

docs/configuring-polaris-for-production.md

@@ -62,13 +62,13 @@ Before using Polaris when using a metastore manager other than `in-memory`, you
 To bootstrap Polaris, run:
 
 ```bash
-java -jar /path/to/jar/polaris-service-1.0.0-all.jar bootstrap polaris-server.yml
+java -jar /path/to/jar/polaris-service-all.jar bootstrap polaris-server.yml
 ```
 
 Afterwards, Polaris can be launched normally:
 
 ```bash
-java -jar /path/to/jar/polaris-service-1.0.0-all.jar server polaris-server.yml
+java -jar /path/to/jar/polaris-service-all.jar server polaris-server.yml
 ```
 
 ## Other Configurations

gradle/wrapper/gradle-wrapper.properties

@@ -17,8 +17,8 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 # See https://gradle.org/release-checksums/ for valid checksums
-distributionSha256Sum=d725d707bfabd4dfdc958c624003b3c80accc03f7037b5122c4b1d0ef15cecab
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip
+distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

polaris-core/src/main/java/io/polaris/core/PolarisConfigurationStore.java

@@ -19,16 +19,12 @@
 import io.polaris.core.entity.CatalogEntity;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Dynamic configuration store used to retrieve runtime parameters, which may vary by realm or by
  * request.
  */
 public interface PolarisConfigurationStore {
-  Logger LOGGER = LoggerFactory.getLogger(PolarisConfigurationStore.class);
-
   /**
    * Retrieve the current value for a configuration key. May be null if not set.
    *
@@ -103,7 +99,6 @@ public interface PolarisConfigurationStore {
       PolarisConfiguration<T> config) {
     if (config.hasCatalogConfig()
         && catalogEntity.getPropertiesAsMap().containsKey(config.catalogConfig())) {
-      LOGGER.debug("Loaded config from catalog: {}", config.catalogConfig());
       return tryCast(config, catalogEntity.getPropertiesAsMap().get(config.catalogConfig()));
     } else {
       return getConfiguration(ctx, config);

polaris-core/src/main/java/io/polaris/core/monitor/MetricRegistryAware.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.polaris.core.monitor;
+
+/** Allows setting a configured instance of {@link PolarisMetricRegistry} */
+public interface MetricRegistryAware {
+  void setMetricRegistry(PolarisMetricRegistry metricRegistry);
+}

polaris-core/src/main/java/io/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java

@@ -34,6 +34,7 @@
 import java.util.function.Supplier;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The common implementation of Configuration interface for configuring the {@link
@@ -51,8 +52,8 @@ public abstract class LocalPolarisMetaStoreManagerFactory<StoreType>
 
   protected PolarisStorageIntegrationProvider storageIntegration;
 
-  private final Logger LOGGER =
-      org.slf4j.LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class);
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class);
 
   protected abstract StoreType createBackingStore(@NotNull PolarisDiagnostics diagnostics);
 
@@ -89,6 +90,22 @@ public synchronized Map<String, PolarisMetaStoreManager.PrincipalSecretsResult>
     return results;
   }
 
+  @Override
+  public void purgeRealms(List<String> realms) {
+    for (String realm : realms) {
+      PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(() -> realm);
+      PolarisMetaStoreSession session = getOrCreateSessionSupplier(() -> realm).get();
+
+      PolarisCallContext callContext = new PolarisCallContext(session, diagServices);
+      metaStoreManager.purge(callContext);
+
+      storageCredentialCacheMap.remove(realm);
+      backingStoreMap.remove(realm);
+      sessionSupplierMap.remove(realm);
+      metaStoreManagerMap.remove(realm);
+    }
+  }
+
   @Override
   public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager(
       RealmContext realmContext) {
@@ -147,6 +164,21 @@ public void setStorageIntegrationProvider(PolarisStorageIntegrationProvider stor
             sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), diagServices);
     CallContext.setCurrentContext(CallContext.of(realmContext, polarisContext));
 
+    PolarisMetaStoreManager.EntityResult preliminaryRootPrincipalLookup =
+        metaStoreManager.readEntityByName(
+            polarisContext,
+            null,
+            PolarisEntityType.PRINCIPAL,
+            PolarisEntitySubType.NULL_SUBTYPE,
+            PolarisEntityConstants.getRootPrincipalName());
+    if (preliminaryRootPrincipalLookup.isSuccess()) {
+      String overrideMessage =
+          "It appears this metastore manager has already been bootstrapped. "
+              + "To continue bootstrapping, please first purge the metastore with the `purge` command.";
+      LOGGER.error("\n\n {} \n\n", overrideMessage);
+      throw new IllegalArgumentException(overrideMessage);
+    }
+
     metaStoreManager.bootstrapPolarisService(polarisContext);
 
     PolarisMetaStoreManager.EntityResult rootPrincipalLookup =

polaris-core/src/main/java/io/polaris/core/persistence/MetaStoreManagerFactory.java

@@ -43,4 +43,7 @@ public interface MetaStoreManagerFactory extends Discoverable {
   void setMetricRegistry(PolarisMetricRegistry metricRegistry);
 
   Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> bootstrapRealms(List<String> realms);
+
+  /** Purge all metadata for the realms provided */
+  void purgeRealms(List<String> realms);
 }

polaris-core/src/main/java/io/polaris/core/persistence/PolarisMetaStoreManager.java

@@ -176,18 +176,28 @@ public boolean alreadyExists() {
   }
 
   /**
-   * Bootstrap the Polaris service, will remove ALL existing persisted entities, then will create
-   * the root catalog, root principal and associated service admin role.
+   * Bootstrap the Polaris service, creating the root catalog, root principal, and associated
+   * service admin role. Will fail if the service has already been bootstrapped.
+   *
+   * @param callCtx call context
+   * @return the result of the bootstrap attempt
+   */
+  @NotNull
+  BaseResult bootstrapPolarisService(@NotNull PolarisCallContext callCtx);
+
+  /**
+   * Purge all metadata associated with the Polaris service, resetting the metastore to the state it
+   * was in prior to bootstrapping.
    *
    * <p>*************************** WARNING ************************
    *
-   * <p>This will destroy whatever Polaris metadata exists in this account
+   * <p>This will destroy whatever Polaris metadata exists in the metastore
    *
    * @param callCtx call context
    * @return always success or unexpected error
    */
   @NotNull
-  BaseResult bootstrapPolarisService(@NotNull PolarisCallContext callCtx);
+  BaseResult purge(@NotNull PolarisCallContext callCtx);
 
   /** the return for an entity lookup call */
   class EntityResult extends BaseResult {

polaris-core/src/main/java/io/polaris/core/persistence/PolarisMetaStoreManagerImpl.java

@@ -52,13 +52,16 @@
 import java.util.stream.Collectors;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Default implementation of the Polaris Meta Store Manager. Uses the underlying meta store to store
  * and retrieve all Polaris metadata
  */
 @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
 public class PolarisMetaStoreManagerImpl implements PolarisMetaStoreManager {
+  private static final Logger LOGGER = LoggerFactory.getLogger(PolarisMetaStoreManagerImpl.class);
 
   /** mapper, allows to serialize/deserialize properties to/from JSON */
   private static final ObjectMapper MAPPER = new ObjectMapper();
@@ -634,9 +637,6 @@ private void revokeGrantRecord(
   private void bootstrapPolarisService(
       @NotNull PolarisCallContext callCtx, @NotNull PolarisMetaStoreSession ms) {
 
-    // cleanup everything, start from a blank slate
-    ms.deleteAll(callCtx);
-
     // Create a root container entity that can represent the securable for any top-level grants.
     PolarisBaseEntity rootContainer =
         new PolarisBaseEntity(
@@ -706,6 +706,20 @@ private void bootstrapPolarisService(
     return new BaseResult(ReturnStatus.SUCCESS);
   }
 
+  @Override
+  public @NotNull BaseResult purge(@NotNull PolarisCallContext callCtx) {
+    // get meta store we should be using
+    PolarisMetaStoreSession ms = callCtx.getMetaStore();
+
+    // run operation in a read/write transaction
+    LOGGER.warn("Deleting all metadata in the metastore...");
+    ms.runActionInTransaction(callCtx, () -> ms.deleteAll(callCtx));
+    LOGGER.warn("Finished deleting all metadata in the metastore");
+
+    // all good
+    return new BaseResult(ReturnStatus.SUCCESS);
+  }
+
   /**
    * See {@link #readEntityByName(PolarisCallContext, List, PolarisEntityType, PolarisEntitySubType,
    * String)}

polaris-core/src/main/java/io/polaris/core/persistence/PolarisObjectMapperUtil.java

@@ -35,6 +35,8 @@
 
 /** A mapper to serialize/deserialize polaris objects. */
 public class PolarisObjectMapperUtil {
+  private static final Logger LOGGER = LoggerFactory.getLogger(PolarisObjectMapperUtil.class);
+
   /** mapper, allows to serialize/deserialize properties to/from JSON */
   private static final ObjectMapper MAPPER = configureMapper();
 
@@ -174,8 +176,7 @@ public int getAttemptCount() {
       }
       return new TaskExecutionState(executorId, lastAttemptStartTime, attemptCount);
     } catch (IOException e) {
-      Logger logger = LoggerFactory.getLogger(PolarisObjectMapperUtil.class);
-      logger
+      LOGGER
           .atWarn()
           .addKeyValue("json", entity.getProperties())
           .addKeyValue("error", e.getMessage())

polaris-core/src/main/java/io/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java

@@ -54,7 +54,8 @@
 public class AzureCredentialsStorageIntegration
     extends InMemoryStorageIntegration<AzureStorageConfigurationInfo> {
 
-  private final Logger LOGGER = LoggerFactory.getLogger(AzureCredentialsStorageIntegration.class);
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(AzureCredentialsStorageIntegration.class);
 
   final DefaultAzureCredential defaultAzureCredential;
 

polaris-core/src/main/java/io/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java

@@ -49,7 +49,8 @@
  */
 public class GcpCredentialsStorageIntegration
     extends InMemoryStorageIntegration<GcpStorageConfigurationInfo> {
-  private final Logger LOGGER = LoggerFactory.getLogger(GcpCredentialsStorageIntegration.class);
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(GcpCredentialsStorageIntegration.class);
 
   private final GoogleCredentials sourceCredentials;
   private final HttpTransportFactory transportFactory;

polaris-core/src/test/java/io/polaris/core/storage/cache/StorageCredentialCacheTest.java

@@ -37,6 +37,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import org.apache.iceberg.exceptions.UnprocessableEntityException;
 import org.assertj.core.api.Assertions;
 import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.RepeatedTest;
@@ -94,7 +95,8 @@ public void testBadResult() {
                     true,
                     new HashSet<>(Arrays.asList("s3://bucket1/path")),
                     new HashSet<>(Arrays.asList("s3://bucket3/path"))))
-        .isInstanceOf(RuntimeException.class);
+        .isInstanceOf(UnprocessableEntityException.class)
+        .hasMessage("Failed to get subscoped credentials: extra_error_info");
   }
 
   @Test

polaris-core/src/test/java/io/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java

@@ -63,7 +63,8 @@ class GcpCredentialsStorageIntegrationTest {
   private final String gcsServiceKeyJsonFileLocation =
       System.getenv("GOOGLE_APPLICATION_CREDENTIALS");
 
-  private final Logger LOGGER = LoggerFactory.getLogger(GcpCredentialsStorageIntegrationTest.class);
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(GcpCredentialsStorageIntegrationTest.class);
 
   @ParameterizedTest
   @ValueSource(booleans = {true, false})

polaris-core/src/testFixtures/java/io/polaris/core/persistence/PolarisTestMetaStoreManager.java

@@ -65,6 +65,7 @@ public PolarisTestMetaStoreManager(
     this.doRetry = false;
 
     // bootstrap the Polaris service
+    polarisMetaStoreManager.purge(polarisCallContext);
     polarisMetaStoreManager.bootstrapPolarisService(polarisCallContext);
   }
 

polaris-server.yml

@@ -89,6 +89,8 @@ metaStoreManager:
   # type: eclipse-link # uncomment to use eclipse-link as metastore
   # persistence-unit: polaris
 
+io:
+  factoryType: default
 
 # TODO - avoid duplicating token broker config
 oauth2: