From 0626694df5b06c64e913b431d5c98b4ab889f72f Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Wed, 9 Nov 2022 23:27:21 +0100 Subject: [PATCH] feat: Add support for whitelisting topic prefixes Closes devshawn/kafka-gitops#90 --- docs/specification.md | 9 +++++ .../devshawn/kafka/gitops/StateManager.java | 18 ++++++++-- .../gitops/domain/state/DesiredState.java | 2 ++ .../domain/state/settings/SettingsTopics.java | 4 ++- ...Blacklist.java => SettingsTopicsList.java} | 6 ++-- .../kafka/gitops/manager/PlanManager.java | 20 +++++++---- .../gitops/PlanCommandIntegrationSpec.groovy | 2 ++ .../seed-blacklist-whitelist-topics-plan.json | 35 +++++++++++++++++++ .../seed-blacklist-whitelist-topics.yaml | 14 ++++++++ .../plans/seed-whitelist-topics-plan.json | 35 +++++++++++++++++++ .../plans/seed-whitelist-topics.yaml | 10 ++++++ 11 files changed, 141 insertions(+), 14 deletions(-) rename src/main/java/com/devshawn/kafka/gitops/domain/state/settings/{SettingsTopicsBlacklist.java => SettingsTopicsList.java} (59%) create mode 100644 src/test/resources/plans/seed-blacklist-whitelist-topics-plan.json create mode 100644 src/test/resources/plans/seed-blacklist-whitelist-topics.yaml create mode 100644 src/test/resources/plans/seed-whitelist-topics-plan.json create mode 100644 src/test/resources/plans/seed-whitelist-topics.yaml diff --git a/docs/specification.md b/docs/specification.md index 3e25f247..f11898e7 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -23,6 +23,11 @@ The desired state file consists of: - **topics** [Optional]: - **defaults** [Optional]: Specify topic defaults so you don't need to specify them for every topic in the state file. Currently, only replication is supported. - **blacklist** [Optional]: Add a prefixed topic blacklist for ignoring specific topics when using `kafka-gitops`. This allows topics to be ignored from being deleted if they are not defined in the desired state file. + - **whitelist** [Optional]: Add a prefixed topic whitelist for exclusively handling specific topics when using `kafka-gitops`. This allows topics to be exclusively handled and topics not on the list are being ignored, even if they are not defined in the desired state file. + +?> `topics.blacklist` and `topics.whitelist` are _not mutually exclusive_ can be used together to whitelist specific topic prefixes and blacklist individual "sub-topics". + +?> The blacklist takes precedence over the whitelist, so if a topic name is matched by both, it will be ignored and not deleted if it was not defined in the desired state file. **Example**: ```yaml @@ -35,6 +40,10 @@ settings: blacklist: prefixed: - _confluent + - my-topics-excluded + whitelist: + prefixed: + - my-topics ``` ## Topics diff --git a/src/main/java/com/devshawn/kafka/gitops/StateManager.java b/src/main/java/com/devshawn/kafka/gitops/StateManager.java index c991731c..3ffaa476 100644 --- a/src/main/java/com/devshawn/kafka/gitops/StateManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/StateManager.java @@ -18,7 +18,7 @@ import com.devshawn.kafka.gitops.domain.state.settings.Settings; import com.devshawn.kafka.gitops.domain.state.settings.SettingsCCloud; import com.devshawn.kafka.gitops.domain.state.settings.SettingsTopics; -import com.devshawn.kafka.gitops.domain.state.settings.SettingsTopicsBlacklist; +import com.devshawn.kafka.gitops.domain.state.settings.SettingsTopicsList; import com.devshawn.kafka.gitops.exception.ConfluentCloudException; import com.devshawn.kafka.gitops.exception.InvalidAclDefinitionException; import com.devshawn.kafka.gitops.exception.MissingConfigurationException; @@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -153,7 +154,8 @@ private void createServiceAccount(String name, List serviceAccou private DesiredState getDesiredState() { DesiredStateFile desiredStateFile = getAndValidateStateFile(); DesiredState.Builder desiredState = new DesiredState.Builder() - .addAllPrefixedTopicsToIgnore(getPrefixedTopicsToIgnore(desiredStateFile)); + .addAllPrefixedTopicsToIgnore(getPrefixedTopicsToIgnore(desiredStateFile)) + .addAllPrefixedTopicsToAccept(getPrefixedTopicsToAccept(desiredStateFile)); generateTopicsState(desiredState, desiredStateFile); @@ -297,7 +299,7 @@ private List getPrefixedTopicsToIgnore(DesiredStateFile desiredStateFile desiredStateFile.getSettings() .flatMap(Settings::getTopics) .flatMap(SettingsTopics::getBlacklist) - .map(SettingsTopicsBlacklist::getPrefixed) + .map(SettingsTopicsList::getPrefixed) .ifPresent(topics::addAll); desiredStateFile.getServices().forEach((name, service) -> { @@ -308,6 +310,16 @@ private List getPrefixedTopicsToIgnore(DesiredStateFile desiredStateFile return topics; } + private List getPrefixedTopicsToAccept(DesiredStateFile desiredStateFile) { + return desiredStateFile.getSettings() + .flatMap(Settings::getTopics) + .flatMap(SettingsTopics::getWhitelist) + .map(SettingsTopicsList::getPrefixed) + .stream() + .flatMap(Collection::stream) + .toList(); + } + private GetAclOptions buildGetAclOptions(String serviceName) { return new GetAclOptions.Builder().setServiceName(serviceName).setDescribeAclEnabled(describeAclEnabled).build(); } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java index d46d6d78..33067a8f 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java @@ -16,6 +16,8 @@ public interface DesiredState { List getPrefixedTopicsToIgnore(); + List getPrefixedTopicsToAccept(); + class Builder extends DesiredState_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopics.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopics.java index 985e3547..3db186c4 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopics.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopics.java @@ -11,7 +11,9 @@ public interface SettingsTopics { Optional getDefaults(); - Optional getBlacklist(); + Optional getBlacklist(); + + Optional getWhitelist(); class Builder extends SettingsTopics_Builder { } diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsBlacklist.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsList.java similarity index 59% rename from src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsBlacklist.java rename to src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsList.java index 2dcbede4..75048138 100644 --- a/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsBlacklist.java +++ b/src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsList.java @@ -6,11 +6,11 @@ import java.util.List; @FreeBuilder -@JsonDeserialize(builder = SettingsTopicsBlacklist.Builder.class) -public interface SettingsTopicsBlacklist { +@JsonDeserialize(builder = SettingsTopicsList.Builder.class) +public interface SettingsTopicsList { List getPrefixed(); - class Builder extends SettingsTopicsBlacklist_Builder { + class Builder extends SettingsTopicsList_Builder { } } diff --git a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java index 726cc1d6..2a356873 100644 --- a/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java +++ b/src/main/java/com/devshawn/kafka/gitops/manager/PlanManager.java @@ -66,14 +66,20 @@ public void planTopics(DesiredState desiredState, DesiredPlan.Builder desiredPla desiredPlan.addTopicPlans(topicPlan.build()); }); - topics.forEach(currentTopic -> { - boolean shouldIgnore = desiredState.getPrefixedTopicsToIgnore().stream().anyMatch(it -> currentTopic.name().startsWith(it)); - if (shouldIgnore) { - LOG.info("[PLAN] Ignoring topic {} due to prefix", currentTopic.name()); - return; + for (TopicListing currentTopic : topics) { + boolean acceptTopic = desiredState.getPrefixedTopicsToAccept().stream().anyMatch(it -> currentTopic.name().startsWith(it)); + if (!desiredState.getPrefixedTopicsToAccept().isEmpty() && !acceptTopic) { + LOG.info("[PLAN] Ignoring topic {} due to missing prefix (whitelist)", currentTopic.name()); + continue; } - if (!managerConfig.isDeleteDisabled() && desiredState.getTopics().getOrDefault(currentTopic.name(), null) == null) { + boolean ignoreTopic = desiredState.getPrefixedTopicsToIgnore().stream().anyMatch(it -> currentTopic.name().startsWith(it)); + if (ignoreTopic) { + LOG.info("[PLAN] Ignoring topic {} due to prefix (blacklist)", currentTopic.name()); + continue; + } + + if (!managerConfig.isDeleteDisabled() && !desiredState.getTopics().containsKey(currentTopic.name())) { TopicPlan topicPlan = new TopicPlan.Builder() .setName(currentTopic.name()) .setAction(PlanAction.REMOVE) @@ -81,7 +87,7 @@ public void planTopics(DesiredState desiredState, DesiredPlan.Builder desiredPla desiredPlan.addTopicPlans(topicPlan); } - }); + } } private void planTopicConfigurations(String topicName, TopicDetails topicDetails, List configs, TopicPlan.Builder topicPlan) { diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy index 6d75733c..9cbb6cab 100644 --- a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy +++ b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy @@ -158,6 +158,8 @@ class PlanCommandIntegrationSpec extends Specification { "seed-topic-modification-no-delete" | true "seed-acl-exists" | true "seed-blacklist-topics" | false + "seed-blacklist-whitelist-topics" | false + "seed-whitelist-topics" | false } void 'test include unchanged flag - #planName #includeUnchanged'() { diff --git a/src/test/resources/plans/seed-blacklist-whitelist-topics-plan.json b/src/test/resources/plans/seed-blacklist-whitelist-topics-plan.json new file mode 100644 index 00000000..64162953 --- /dev/null +++ b/src/test/resources/plans/seed-blacklist-whitelist-topics-plan.json @@ -0,0 +1,35 @@ +{ + "topicPlans": [ + { + "name": "new-topic", + "action": "ADD", + "topicDetails": { + "partitions": 6, + "replication": 1, + "configs": {} + }, + "topicConfigPlans": [] + }, + { + "name": "topic-with-configs-1", + "action": "REMOVE", + "topicDetails": null, + "topicConfigPlans": [] + } + ], + "aclPlans": [ + { + "name": "Unnamed ACL", + "aclDetails": { + "name": "test-topic", + "type": "TOPIC", + "pattern": "LITERAL", + "principal": "User:test", + "host": "*", + "operation": "READ", + "permission": "ALLOW" + }, + "action": "REMOVE" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/plans/seed-blacklist-whitelist-topics.yaml b/src/test/resources/plans/seed-blacklist-whitelist-topics.yaml new file mode 100644 index 00000000..5383f73a --- /dev/null +++ b/src/test/resources/plans/seed-blacklist-whitelist-topics.yaml @@ -0,0 +1,14 @@ +settings: + topics: + blacklist: + prefixed: + - test + - topic-with-configs-2 + whitelist: + prefixed: + - topic-with + +topics: + new-topic: + partitions: 6 + replication: 1 \ No newline at end of file diff --git a/src/test/resources/plans/seed-whitelist-topics-plan.json b/src/test/resources/plans/seed-whitelist-topics-plan.json new file mode 100644 index 00000000..cb683f1a --- /dev/null +++ b/src/test/resources/plans/seed-whitelist-topics-plan.json @@ -0,0 +1,35 @@ +{ + "topicPlans": [ + { + "name": "test-new-topic", + "action": "ADD", + "topicDetails": { + "partitions": 6, + "replication": 1, + "configs": {} + }, + "topicConfigPlans": [] + }, + { + "name": "test-topic", + "action": "REMOVE", + "topicDetails": null, + "topicConfigPlans": [] + } + ], + "aclPlans": [ + { + "name": "Unnamed ACL", + "aclDetails": { + "name": "test-topic", + "type": "TOPIC", + "pattern": "LITERAL", + "principal": "User:test", + "host": "*", + "operation": "READ", + "permission": "ALLOW" + }, + "action": "REMOVE" + } + ] +} diff --git a/src/test/resources/plans/seed-whitelist-topics.yaml b/src/test/resources/plans/seed-whitelist-topics.yaml new file mode 100644 index 00000000..07dfe4b6 --- /dev/null +++ b/src/test/resources/plans/seed-whitelist-topics.yaml @@ -0,0 +1,10 @@ +settings: + topics: + whitelist: + prefixed: + - test + +topics: + test-new-topic: + partitions: 6 + replication: 1