Skip to content

Commit

Permalink
Polaris Admin Tool
Browse files Browse the repository at this point in the history
  • Loading branch information
adutra committed Jan 11, 2025
1 parent b8d029d commit 1571116
Show file tree
Hide file tree
Showing 25 changed files with 694 additions and 60 deletions.
2 changes: 2 additions & 0 deletions LICENSE-BINARY-DIST
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ commons-io:commons-io
commons-logging:commons-logging
commons-net:commons-net
dev.failsafe:failsafe
info.picocli:picocli
io.airlift:aircompressor
io.grpc:grpc-alts
io.grpc:grpc-api
Expand Down Expand Up @@ -400,6 +401,7 @@ io.quarkus:quarkus-micrometer-registry-prometheus
io.quarkus:quarkus-mutiny
io.quarkus:quarkus-netty
io.quarkus:quarkus-opentelemetry
io.quarkus:quarkus-picocli
io.quarkus:quarkus-reactive-routes
io.quarkus:quarkus-rest
io.quarkus:quarkus-rest-common
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@

import io.smallrye.common.annotation.Identifier;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.nio.file.Path;
import org.apache.polaris.core.PolarisDiagnostics;
import org.apache.polaris.core.context.RealmContext;
import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory;
import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;
Expand All @@ -50,14 +52,16 @@ protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics

@Override
protected PolarisMetaStoreSession createMetaStoreSession(
@Nonnull PolarisEclipseLinkStore store, @Nonnull RealmContext realmContext) {
@Nonnull PolarisEclipseLinkStore store,
@Nonnull RealmContext realmContext,
@Nullable PolarisCredentialsBootstrap credentialsBootstrap) {
return new PolarisEclipseLinkMetaStoreSessionImpl(
store,
storageIntegrationProvider,
realmContext,
configurationFile(),
persistenceUnitName(),
secretsGenerator(realmContext));
secretsGenerator(realmContext, credentialsBootstrap));
}

private String configurationFile() {
Expand Down
1 change: 1 addition & 0 deletions gradle/projects.main.properties
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ polaris-api-management-service=api/management-service
polaris-service-common=service/common
polaris-quarkus-service=quarkus/service
polaris-quarkus-server=quarkus/server
polaris-quarkus-admin=quarkus/admin
polaris-eclipselink=extension/persistence/eclipselink
polaris-jpa-model=extension/persistence/jpa-model
aggregated-license-report=aggregated-license-report
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.polaris.core.persistence;

import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -57,50 +58,49 @@ public abstract class LocalPolarisMetaStoreManagerFactory<StoreType>
private static final Logger LOGGER =
LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class);

private boolean bootstrap;

protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics);

protected abstract PolarisMetaStoreSession createMetaStoreSession(
@Nonnull StoreType store, @Nonnull RealmContext realmContext);
@Nonnull StoreType store,
@Nonnull RealmContext realmContext,
@Nullable PolarisCredentialsBootstrap credentialsBootstrap);

protected PrincipalSecretsGenerator secretsGenerator(RealmContext realmContext) {
if (bootstrap) {
return PrincipalSecretsGenerator.bootstrap(realmContext.getRealmIdentifier());
protected PrincipalSecretsGenerator secretsGenerator(
RealmContext realmContext, @Nullable PolarisCredentialsBootstrap credentialsBootstrap) {
if (credentialsBootstrap != null) {
return PrincipalSecretsGenerator.bootstrap(
realmContext.getRealmIdentifier(), credentialsBootstrap);
} else {
return PrincipalSecretsGenerator.RANDOM_SECRETS;
}
}

private void initializeForRealm(RealmContext realmContext) {
private void initializeForRealm(
RealmContext realmContext, PolarisCredentialsBootstrap credentialsBootstrap) {
final StoreType backingStore = createBackingStore(diagServices);
backingStoreMap.put(realmContext.getRealmIdentifier(), backingStore);
sessionSupplierMap.put(
realmContext.getRealmIdentifier(),
() -> createMetaStoreSession(backingStore, realmContext));
() -> createMetaStoreSession(backingStore, realmContext, credentialsBootstrap));

PolarisMetaStoreManager metaStoreManager = new PolarisMetaStoreManagerImpl();
metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager);
}

@Override
public synchronized Map<String, PrincipalSecretsResult> bootstrapRealms(List<String> realms) {
public synchronized Map<String, PrincipalSecretsResult> bootstrapRealms(
List<String> realms, PolarisCredentialsBootstrap credentialsBootstrap) {
Map<String, PrincipalSecretsResult> results = new HashMap<>();

bootstrap = true;
try {
for (String realm : realms) {
RealmContext realmContext = () -> realm;
if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) {
initializeForRealm(realmContext);
PrincipalSecretsResult secretsResult =
bootstrapServiceAndCreatePolarisPrincipalForRealm(
realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
results.put(realmContext.getRealmIdentifier(), secretsResult);
}
for (String realm : realms) {
RealmContext realmContext = () -> realm;
if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) {
initializeForRealm(realmContext, credentialsBootstrap);
PrincipalSecretsResult secretsResult =
bootstrapServiceAndCreatePolarisPrincipalForRealm(
realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
results.put(realmContext.getRealmIdentifier(), secretsResult);
}
} finally {
bootstrap = false;
}

return results;
Expand All @@ -126,7 +126,7 @@ public void purgeRealms(List<String> realms) {
public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager(
RealmContext realmContext) {
if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) {
initializeForRealm(realmContext);
initializeForRealm(realmContext, null);
checkPolarisServiceBootstrappedForRealm(
realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
}
Expand All @@ -137,7 +137,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager(
public synchronized Supplier<PolarisMetaStoreSession> getOrCreateSessionSupplier(
RealmContext realmContext) {
if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) {
initializeForRealm(realmContext);
initializeForRealm(realmContext, null);
checkPolarisServiceBootstrappedForRealm(
realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ public interface MetaStoreManagerFactory {

EntityCache getOrCreateEntityCache(RealmContext realmContext);

Map<String, PrincipalSecretsResult> bootstrapRealms(List<String> realms);
Map<String, PrincipalSecretsResult> bootstrapRealms(
List<String> realms, PolarisCredentialsBootstrap credentialsBootstrap);

/** Purge all metadata for the realms provided */
void purgeRealms(List<String> realms);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
*/
public class PolarisCredentialsBootstrap {

public static final PolarisCredentialsBootstrap EMPTY =
new PolarisCredentialsBootstrap(new HashMap<>());

/**
* Parse credentials from the system property {@code polaris.bootstrap.credentials} or the
* environment variable {@code POLARIS_BOOTSTRAP_CREDENTIALS}, whichever is set.
Expand All @@ -55,35 +58,38 @@ public static PolarisCredentialsBootstrap fromEnvironment() {
* </pre>
*/
public static PolarisCredentialsBootstrap fromString(@Nullable String credentialsString) {
return credentialsString != null && !credentialsString.isBlank()
? fromList(Splitter.on(';').trimResults().splitToList(credentialsString))
: EMPTY;
}

/**
* Parse a list of credentials; each element should be in the format: {@code
* realm,principal,clientId,clientSecret}.
*/
public static PolarisCredentialsBootstrap fromList(List<String> credentialsList) {
Map<String, Map<String, Map.Entry<String, String>>> credentials = new HashMap<>();
if (credentialsString != null && !credentialsString.isBlank()) {
Splitter.on(';')
.trimResults()
.splitToList(credentialsString)
.forEach(
quadruple -> {
if (!quadruple.isBlank()) {
List<String> parts = Splitter.on(',').trimResults().splitToList(quadruple);
if (parts.size() != 4) {
throw new IllegalArgumentException("Invalid credentials format: " + quadruple);
}
String realmName = parts.get(0);
String principalName = parts.get(1);
String clientId = parts.get(2);
String clientSecret = parts.get(3);
credentials
.computeIfAbsent(realmName, k -> new HashMap<>())
.merge(
principalName,
new SimpleEntry<>(clientId, clientSecret),
(a, b) -> {
throw new IllegalArgumentException(
"Duplicate principal: " + principalName);
});
}
});
for (String quadruple : credentialsList) {
if (!quadruple.isBlank()) {
List<String> parts = Splitter.on(',').trimResults().splitToList(quadruple);
if (parts.size() != 4) {
throw new IllegalArgumentException("Invalid credentials format: " + quadruple);
}
String realmName = parts.get(0);
String principalName = parts.get(1);
String clientId = parts.get(2);
String clientSecret = parts.get(3);
credentials
.computeIfAbsent(realmName, k -> new HashMap<>())
.merge(
principalName,
new SimpleEntry<>(clientId, clientSecret),
(a, b) -> {
throw new IllegalArgumentException("Duplicate principal: " + principalName);
});
}
}
return new PolarisCredentialsBootstrap(credentials);
return credentials.isEmpty() ? EMPTY : new PolarisCredentialsBootstrap(credentials);
}

@VisibleForTesting final Map<String, Map<String, Map.Entry<String, String>>> credentials;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import java.util.Comparator;
import java.util.List;
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -87,6 +88,27 @@ void getSecretsValidString() {
.contains(new PolarisPrincipalSecrets(123, "client2a", "secret2a", "secret2a"));
}

@Test
void getSecretsValidList() {
PolarisCredentialsBootstrap credentials =
PolarisCredentialsBootstrap.fromList(
List.of(
"realm1,user1a,client1a,secret1a",
"realm1,user1b,client1b,secret1b",
"realm2,user2a,client2a,secret2a"));
assertThat(credentials.getSecrets("realm1", 123, "nonexistent")).isEmpty();
assertThat(credentials.getSecrets("nonexistent", 123, "user1a")).isEmpty();
assertThat(credentials.getSecrets("realm1", 123, "user1a"))
.usingValueComparator(comparator)
.contains(new PolarisPrincipalSecrets(123, "client1a", "secret1a", "secret1a"));
assertThat(credentials.getSecrets("realm1", 123, "user1b"))
.usingValueComparator(comparator)
.contains(new PolarisPrincipalSecrets(123, "client1b", "secret1b", "secret1b"));
assertThat(credentials.getSecrets("realm2", 123, "user2a"))
.usingValueComparator(comparator)
.contains(new PolarisPrincipalSecrets(123, "client2a", "secret2a", "secret2a"));
}

@Test
void getSecretsValidSystemProperty() {
PolarisCredentialsBootstrap credentials = PolarisCredentialsBootstrap.fromEnvironment();
Expand Down
27 changes: 27 additions & 0 deletions quarkus/admin/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Polaris Admin Tool

This module contains a maintenance tool for performing administrative tasks on the Polaris database.
It is a Quarkus application that can be used to perform various maintenance tasks targeting the
Polaris database directly.

Building this module will create a runnable uber-jar that can be executed from the command line.

To also build the Docker image, you can use the following command:

```shell
./gradlew :polaris-quarkus-admin:assemble -Dquarkus.container-image.build=true
```

## Running the Admin Tool

The admin tool can be run from the command line using the following command:

```shell
java -jar polaris-quarkus-admin-<version>-runner.jar --help
```

Using the Docker image, you can run the admin tool with the following command:

```shell
docker run --rm -it polaris-admin-tool:<version> --help
```
86 changes: 86 additions & 0 deletions quarkus/admin/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import io.quarkus.gradle.tasks.QuarkusBuild

plugins {
alias(libs.plugins.quarkus)
alias(libs.plugins.openapi.generator)
id("polaris-server")
id("polaris-license-report")
}

dependencies {
implementation(project(":polaris-core"))
implementation(project(":polaris-api-management-service"))
implementation(project(":polaris-api-iceberg-service"))
implementation(project(":polaris-service-common"))
implementation(project(":polaris-quarkus-service"))

implementation(platform(libs.quarkus.bom))
implementation("io.quarkus:quarkus-picocli")
implementation("io.quarkus:quarkus-container-image-docker")

implementation("org.jboss.slf4j:slf4j-jboss-logmanager")

// override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329
implementation(platform(libs.dnsjava))

testImplementation(enforcedPlatform(libs.quarkus.bom))
testImplementation("io.quarkus:quarkus-junit5")

testImplementation(platform(libs.junit.bom))
testImplementation(libs.bundles.junit.testing)
}

tasks.withType<ProcessResources>().configureEach {
from("src/main/resources") {
expand("polarisVersion" to version)
duplicatesStrategy = DuplicatesStrategy.INCLUDE
}
}

quarkus {
quarkusBuildProperties.put("quarkus.package.type", "uber-jar")
// Pull manifest attributes from the "main" `jar` task to get the
// release-information into the jars generated by Quarkus.
quarkusBuildProperties.putAll(
provider {
tasks
.named("jar", Jar::class.java)
.get()
.manifest
.attributes
.map { e -> "quarkus.package.jar.manifest.attributes.\"${e.key}\"" to e.value.toString() }
.toMap()
}
)
}

publishing {
publications {
named<MavenPublication>("maven") {
val quarkusBuild = tasks.getByName<QuarkusBuild>("quarkusBuild")
artifact(quarkusBuild.runnerJar) {
classifier = "runner"
builtBy(quarkusBuild)
}
}
}
}
Loading

0 comments on commit 1571116

Please sign in to comment.