Fix acceptance test setup, use native Kafka docker images (#439)

Dockerfile.kafka

@@ -1,21 +1,24 @@
-FROM confluentinc/cp-kafka:latest
+FROM apache/kafka-native:3.8.0
 
 ARG broker_id
-ARG zookeeper_connect
 ARG listener_host
 ARG listener_port
 
 COPY secrets/ /etc/kafka/secrets
 
 ENV KAFKA_BROKER_ID=$broker_id
+ENV KAFKA_NODE_ID=$broker_id
 ENV KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
-ENV KAFKA_ZOOKEEPER_CONNECT=$zookeeper_connect
+ENV KAFKA_PROCESS_ROLES='broker,controller'
+ENV KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093
+ENV KAFKA_CONTROLLER_LISTENER_NAMES='CONTROLLER'
+
 
 # confluent's bash script looks for an 'SSL' suffix in listener names:
 # https://github.com/confluentinc/cp-docker-images/blob/76d786d0243ea16626b8b46dba34ec0b1066de84/debian/kafka/include/etc/confluent/docker/configure#L65
 ENV KAFKA_LISTENERS=INTERNAL_SSL://$listener_host:9090,EXTERNAL_SSL://$listener_host:9092
 ENV KAFKA_ADVERTISED_LISTENERS=INTERNAL_SSL://$listener_host:9090,EXTERNAL_SSL://localhost:$listener_port
-ENV KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_SSL:SSL,EXTERNAL_SSL:SSL
+ENV KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_SSL:SSL,EXTERNAL_SSL:SSL,CONTROLLER:PLAINTEXT
 ENV KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL_SSL
 
 ENV KAFKA_SSL_KEYSTORE_FILENAME=kafka.$listener_host.keystore.jks
@@ -29,4 +32,9 @@ ENV KAFKA_SSL_CLIENT_AUTH=required
 ENV KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM=
 ENV KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM=
 ENV KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND=true
-ENV KAFKA_AUTHORIZER_CLASS_NAME=kafka.security.authorizer.AclAuthorizer
+ENV KAFKA_AUTHORIZER_CLASS_NAME=org.apache.kafka.metadata.authorizer.StandardAuthorizer
+
+ENV KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+ENV KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
+ENV KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+ENV KAFKA_LOG4J_LOGGERS='org.apache.kafka.image.loader.MetadataLoader=WARN'

docker-compose.yaml

@@ -1,60 +1,38 @@
 ---
-version: '3.2'
+version: "3.2"
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:latest
-    ports:
-      - "2181:2181"
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-    extra_hosts:
-      - "moby:127.0.0.1"
-
   kafka1:
     build:
       context: .
       dockerfile: Dockerfile.kafka
       args:
         broker_id: 1
-        zookeeper_connect: zookeeper:2181
         listener_host: kafka1
         listener_port: 9092
+    environment:
+      CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk"
+      KAFKA_LISTENERS: INTERNAL_SSL://kafka1:9090,EXTERNAL_SSL://kafka1:9092,CONTROLLER://kafka1:9093
     ports:
       - "9092:9092"
-    depends_on:
-      - zookeeper
-    extra_hosts:
-      - "moby:127.0.0.1"
-
   kafka2:
     build:
       context: .
       dockerfile: Dockerfile.kafka
       args:
         broker_id: 2
-        zookeeper_connect: zookeeper:2181
         listener_host: kafka2
-        listener_port: 9093
-    ports:
-      - "9093:9092"
-    depends_on:
-      - zookeeper
-    extra_hosts:
-      - "moby:127.0.0.1"
-
+        listener_port: 9092
+    environment:
+      CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk"
+      KAFKA_PROCESS_ROLES: broker
   kafka3:
     build:
       context: .
       dockerfile: Dockerfile.kafka
       args:
         broker_id: 3
-        zookeeper_connect: zookeeper:2181
         listener_host: kafka3
-        listener_port: 9094
-    ports:
-      - "9094:9092"
-    depends_on:
-      - zookeeper
-    extra_hosts:
-      - "moby:127.0.0.1"
+        listener_port: 9092
+    environment:
+      CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk"
+      KAFKA_PROCESS_ROLES: broker

kafka/provider_test.go

@@ -70,6 +70,7 @@ func accTestProviderConfig() (*terraform.ResourceConfig, error) {
 
 	raw := map[string]interface{}{
 		"bootstrap_servers": bs,
+		"kafka_version":     "3.8.0",
 	}
 
 	return terraform.NewResourceConfigRaw(raw), nil