chore: add metrics docs generation

Makefile

@@ -24,6 +24,7 @@ all: clean build test
 
 clean:
 	./gradlew clean
+	rm config.rst metrics.rst
 
 checkstyle:
 	./gradlew checkstyleMain checkstyleTest checkstyleIntegrationTest
@@ -43,9 +44,14 @@ storage/azure/build/distributions/azure-$(VERSION).tgz:
 	./gradlew build :storage:azure:distTar -x test -x integrationTest -x e2e:test
 
 .PHONY: docs
-docs:
+docs: config.rst metrics.rst
+
+config.rst:
 	./gradlew :docs:genConfigDocs
 
+metrics.rst:
+	./gradlew :docs:genMetricsDocs
+
 test: build
 	./gradlew test -x e2e:test
 

config.rst

@@ -1,6 +1,9 @@
 =================
-RemoteStorageManagerConfig
+Core components
 =================
+-----------------
+RemoteStorageManagerConfig
+-----------------
 ``chunk.size``
   Segment files are chunked into smaller parts to allow for faster processing (e.g. encryption, compression) and for range-fetching. It is recommended to benchmark this value, starting with 4MiB.
 
@@ -91,9 +94,9 @@ RemoteStorageManagerConfig
   * Importance: low
 
 
-=================
+-----------------
 SegmentManifestCacheConfig
-=================
+-----------------
 Under ``fetch.manifest.cache.``
 
 ``retention.ms``
@@ -129,9 +132,9 @@ Under ``fetch.manifest.cache.``
   * Importance: low
 
 
-=================
+-----------------
 SegmentIndexesCacheConfig
-=================
+-----------------
 Under ``fetch.indexes.cache.``
 
 ``retention.ms``
@@ -167,9 +170,9 @@ Under ``fetch.indexes.cache.``
   * Importance: low
 
 
-=================
+-----------------
 ChunkManagerFactoryConfig
-=================
+-----------------
 ``fetch.chunk.cache.class``
   Chunk cache implementation. There are 2 implementations included: io.aiven.kafka.tieredstorage.fetch.cache.MemoryChunkCache and io.aiven.kafka.tieredstorage.fetch.cache.DiskChunkCache
 
@@ -179,9 +182,9 @@ ChunkManagerFactoryConfig
   * Importance: medium
 
 
-=================
+-----------------
 MemoryChunkCacheConfig
-=================
+-----------------
 Under ``fetch.chunk.cache.``
 
 ``size``
@@ -224,9 +227,9 @@ Under ``fetch.chunk.cache.``
   * Importance: low
 
 
-=================
+-----------------
 DiskChunkCacheConfig
-=================
+-----------------
 Under ``fetch.chunk.cache.``
 
 ``path``

core/src/main/java/io/aiven/kafka/tieredstorage/fetch/cache/ChunkCache.java

@@ -47,8 +47,8 @@
 import com.github.benmanes.caffeine.cache.Weigher;
 
 public abstract class ChunkCache<T> implements ChunkManager, Configurable {
-    private static final String METRIC_GROUP = "chunk-cache-metrics";
-    private static final String THREAD_POOL_METRIC_GROUP = "chunk-cache-thread-pool-metrics";
+    public static final String METRIC_GROUP = "chunk-cache-metrics";
+    public static final String THREAD_POOL_METRIC_GROUP = "chunk-cache-thread-pool-metrics";
 
     private final ChunkManager chunkManager;
     private ExecutorService executor;

core/src/main/java/io/aiven/kafka/tieredstorage/fetch/index/MemorySegmentIndexesCache.java

@@ -50,8 +50,8 @@ public class MemorySegmentIndexesCache implements SegmentIndexesCache {
     private static final Logger log = LoggerFactory.getLogger(MemorySegmentIndexesCache.class);
 
     private static final long DEFAULT_MAX_SIZE_BYTES = 10 * 1024 * 1024;
-    private static final String METRIC_GROUP = "segment-indexes-cache-metrics";
-    private static final String THREAD_POOL_METRIC_GROUP = "segment-indexes-cache-thread-pool-metrics";
+    public static final String METRIC_GROUP = "segment-indexes-cache-metrics";
+    public static final String THREAD_POOL_METRIC_GROUP = "segment-indexes-cache-thread-pool-metrics";
 
     private final CaffeineStatsCounter statsCounter = new CaffeineStatsCounter(METRIC_GROUP);
 

core/src/main/java/io/aiven/kafka/tieredstorage/fetch/manifest/MemorySegmentManifestCache.java

@@ -46,8 +46,8 @@
 
 public class MemorySegmentManifestCache implements SegmentManifestCache {
     private static final Logger log = LoggerFactory.getLogger(MemorySegmentManifestCache.class);
-    private static final String METRIC_GROUP = "segment-manifest-cache-metrics";
-    private static final String THREAD_POOL_METRIC_GROUP = "segment-manifest-cache-thread-pool-metrics";
+    public static final String METRIC_GROUP = "segment-manifest-cache-metrics";
+    public static final String THREAD_POOL_METRIC_GROUP = "segment-manifest-cache-thread-pool-metrics";
     private static final long DEFAULT_MAX_SIZE = 1000L;
     private static final long DEFAULT_RETENTION_MS = 3_600_000;
 

core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java

@@ -21,6 +21,7 @@
 import org.apache.kafka.common.MetricNameTemplate;
 
 public class CaffeineMetricsRegistry {
+    public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.cache";
 
     static final String CACHE_HITS = "cache-hits";
     static final String CACHE_HITS_TOTAL = CACHE_HITS + "-total";

core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java

@@ -43,6 +43,7 @@
 import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_LOAD_SUCCESS_TIME;
 import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_MISSES;
 import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_SIZE;
+import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.METRIC_CONTEXT;
 
 /**
  * Records cache metrics managed by Caffeine {@code Cache#stats}.
@@ -87,7 +88,7 @@ public CaffeineStatsCounter(final String groupName) {
 
         metrics = new org.apache.kafka.common.metrics.Metrics(
             new MetricConfig(), List.of(reporter), Time.SYSTEM,
-            new KafkaMetricsContext("aiven.kafka.server.tieredstorage.cache")
+            new KafkaMetricsContext(METRIC_CONTEXT)
         );
 
         metricsRegistry = new CaffeineMetricsRegistry(groupName);

core/src/main/java/io/aiven/kafka/tieredstorage/metrics/Metrics.java

@@ -35,6 +35,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.METRIC_CONTEXT;
 import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.OBJECT_UPLOAD;
 import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.OBJECT_UPLOAD_BYTES;
 import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.SEGMENT_COPY_TIME;
@@ -69,7 +70,7 @@ public Metrics(final Time time, final MetricConfig metricConfig) {
             metricConfig,
             List.of(reporter),
             time,
-            new KafkaMetricsContext("aiven.kafka.server.tieredstorage")
+            new KafkaMetricsContext(METRIC_CONTEXT)
         );
 
         metricsRegistry = new MetricsRegistry();

core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java

@@ -25,7 +25,7 @@
 import io.aiven.kafka.tieredstorage.ObjectKeyFactory;
 
 public class MetricsRegistry {
-
+    public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage";
     static final String METRIC_GROUP = "remote-storage-manager-metrics";
     static final String TAG_NAME_OBJECT_TYPE = "object-type";
     static final String[] OBJECT_TYPE_TAG_NAMES = {TAG_NAME_OBJECT_TYPE};

core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java

@@ -29,6 +29,7 @@
 import org.apache.kafka.common.utils.Time;
 
 import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.ACTIVE_THREADS;
+import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG;
 import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.PARALLELISM;
 import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.POOL_SIZE;
 import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.QUEUED_TASK_COUNT;
@@ -51,7 +52,7 @@ public ThreadPoolMonitor(final String groupName, final ExecutorService pool) {
         final JmxReporter reporter = new JmxReporter();
         metrics = new org.apache.kafka.common.metrics.Metrics(
             new MetricConfig(), List.of(reporter), Time.SYSTEM,
-            new KafkaMetricsContext("aiven.kafka.server.tieredstorage.thread-pool")
+            new KafkaMetricsContext(METRIC_CONFIG)
         );
         final var metricsRegistry = new ThreadPoolMonitorMetricsRegistry(groupName);
         registerSensor(metricsRegistry.activeThreadsTotalMetricName, ACTIVE_THREADS, this::activeThreadCount);

core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java

@@ -21,6 +21,7 @@
 import org.apache.kafka.common.MetricNameTemplate;
 
 public class ThreadPoolMonitorMetricsRegistry {
+    public static final String METRIC_CONFIG = "aiven.kafka.server.tieredstorage.thread-pool";
 
     static final String ACTIVE_THREADS = "active-thread-count";
     private static final String ACTIVE_THREADS_TOTAL = ACTIVE_THREADS + "-total";

docs/build.gradle

@@ -31,3 +31,9 @@ tasks.register('genConfigDocs', JavaExec) {
     mainClass = 'io.aiven.kafka.tieredstorage.misc.ConfigDocs'
     standardOutput = new File("config.rst").newOutputStream()
 }
+
+tasks.register('genMetricsDocs', JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    mainClass = 'io.aiven.kafka.tieredstorage.misc.MetricDocs'
+    standardOutput = new File("metrics.rst").newOutputStream()
+}

docs/src/main/java/io/aiven/kafka/tieredstorage/misc/ConfigDocs.java

@@ -39,30 +39,32 @@
  **/
 public class ConfigDocs {
     public static void main(final String[] args) {
-        printSectionTitle("RemoteStorageManagerConfig");
+        printSectionTitle("Core components");
+
+        printSubsectionTitle("RemoteStorageManagerConfig");
         final var rsmConfigDef = RemoteStorageManagerConfig.configDef();
         System.out.println(rsmConfigDef.toEnrichedRst());
 
-        printSectionTitle("SegmentManifestCacheConfig");
+        printSubsectionTitle("SegmentManifestCacheConfig");
         System.out.println("Under ``" + SEGMENT_MANIFEST_CACHE_PREFIX + "``\n");
         final var segmentManifestCacheDef = MemorySegmentManifestCache.configDef();
         System.out.println(segmentManifestCacheDef.toEnrichedRst());
 
-        printSectionTitle("SegmentIndexesCacheConfig");
+        printSubsectionTitle("SegmentIndexesCacheConfig");
         System.out.println("Under ``" + FETCH_INDEXES_CACHE_PREFIX + "``\n");
         final var segmentIndexesCacheDef = MemorySegmentIndexesCache.configDef();
         System.out.println(segmentIndexesCacheDef.toEnrichedRst());
 
-        printSectionTitle("ChunkManagerFactoryConfig");
+        printSubsectionTitle("ChunkManagerFactoryConfig");
         final var chunkCacheFactoryDef = ChunkManagerFactoryConfig.configDef();
         System.out.println(chunkCacheFactoryDef.toEnrichedRst());
 
-        printSectionTitle("MemoryChunkCacheConfig");
+        printSubsectionTitle("MemoryChunkCacheConfig");
         System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
         final var memChunkCacheDef = ChunkCacheConfig.configDef(new ConfigDef());
         System.out.println(memChunkCacheDef.toEnrichedRst());
 
-        printSectionTitle("DiskChunkCacheConfig");
+        printSubsectionTitle("DiskChunkCacheConfig");
         System.out.println("Under ``" + FETCH_CHUNK_CACHE_PREFIX + "``\n");
         final var diskChunkCacheDef = DiskChunkCacheConfig.configDef();
         System.out.println(diskChunkCacheDef.toEnrichedRst());