From 20dc6c28f30f4fd8d83ba0cc651765d2294ab15e Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Mon, 14 Oct 2024 19:13:15 +0300 Subject: [PATCH] chore: add metrics docs generation --- Makefile | 8 +- .../tieredstorage/fetch/cache/ChunkCache.java | 4 +- .../index/MemorySegmentIndexesCache.java | 4 +- .../manifest/MemorySegmentManifestCache.java | 4 +- .../metrics/CaffeineMetricsRegistry.java | 1 + .../metrics/CaffeineStatsCounter.java | 3 +- .../kafka/tieredstorage/metrics/Metrics.java | 3 +- .../metrics/MetricsRegistry.java | 2 +- .../metrics/ThreadPoolMonitor.java | 3 +- .../ThreadPoolMonitorMetricsRegistry.java | 1 + docs/build.gradle | 6 + .../kafka/tieredstorage/misc/MetricDocs.java | 189 +++++++++ metrics.rst | 367 ++++++++++++++++++ .../storage/azure/MetricCollector.java | 3 +- .../storage/azure/MetricRegistry.java | 1 + .../storage/gcs/MetricCollector.java | 3 +- .../storage/gcs/MetricRegistry.java | 1 + .../storage/s3/MetricCollector.java | 3 +- .../storage/s3/MetricRegistry.java | 1 + 19 files changed, 593 insertions(+), 14 deletions(-) create mode 100644 docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricDocs.java create mode 100644 metrics.rst diff --git a/Makefile b/Makefile index 202d7930e..3e8c7df3c 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ all: clean build test clean: ./gradlew clean + rm config.rst metrics.rst checkstyle: ./gradlew checkstyleMain checkstyleTest checkstyleIntegrationTest @@ -43,9 +44,14 @@ storage/azure/build/distributions/azure-$(VERSION).tgz: ./gradlew build :storage:azure:distTar -x test -x integrationTest -x e2e:test .PHONY: docs -docs: +docs: config.rst metrics.rst + +config.rst: ./gradlew :docs:genConfigDocs +metrics.rst: + ./gradlew :docs:genMetricsDocs + test: build ./gradlew test -x e2e:test diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/cache/ChunkCache.java b/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/cache/ChunkCache.java index 9dafcf46e..8c2af76b4 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/cache/ChunkCache.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/cache/ChunkCache.java @@ -47,8 +47,8 @@ import com.github.benmanes.caffeine.cache.Weigher; public abstract class ChunkCache implements ChunkManager, Configurable { - private static final String METRIC_GROUP = "chunk-cache-metrics"; - private static final String THREAD_POOL_METRIC_GROUP = "chunk-cache-thread-pool-metrics"; + public static final String METRIC_GROUP = "chunk-cache-metrics"; + public static final String THREAD_POOL_METRIC_GROUP = "chunk-cache-thread-pool-metrics"; private final ChunkManager chunkManager; private ExecutorService executor; diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/index/MemorySegmentIndexesCache.java b/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/index/MemorySegmentIndexesCache.java index 5bf4f64a8..deb350675 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/index/MemorySegmentIndexesCache.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/index/MemorySegmentIndexesCache.java @@ -50,8 +50,8 @@ public class MemorySegmentIndexesCache implements SegmentIndexesCache { private static final Logger log = LoggerFactory.getLogger(MemorySegmentIndexesCache.class); private static final long DEFAULT_MAX_SIZE_BYTES = 10 * 1024 * 1024; - private static final String METRIC_GROUP = "segment-indexes-cache-metrics"; - private static final String THREAD_POOL_METRIC_GROUP = "segment-indexes-cache-thread-pool-metrics"; + public static final String METRIC_GROUP = "segment-indexes-cache-metrics"; + public static final String THREAD_POOL_METRIC_GROUP = "segment-indexes-cache-thread-pool-metrics"; private final CaffeineStatsCounter statsCounter = new CaffeineStatsCounter(METRIC_GROUP); diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/manifest/MemorySegmentManifestCache.java b/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/manifest/MemorySegmentManifestCache.java index cadf2d804..e4ea3f9f1 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/manifest/MemorySegmentManifestCache.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/fetch/manifest/MemorySegmentManifestCache.java @@ -46,8 +46,8 @@ public class MemorySegmentManifestCache implements SegmentManifestCache { private static final Logger log = LoggerFactory.getLogger(MemorySegmentManifestCache.class); - private static final String METRIC_GROUP = "segment-manifest-cache-metrics"; - private static final String THREAD_POOL_METRIC_GROUP = "segment-manifest-cache-thread-pool-metrics"; + public static final String METRIC_GROUP = "segment-manifest-cache-metrics"; + public static final String THREAD_POOL_METRIC_GROUP = "segment-manifest-cache-thread-pool-metrics"; private static final long DEFAULT_MAX_SIZE = 1000L; private static final long DEFAULT_RETENTION_MS = 3_600_000; diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java index ff2782b01..52f641cfc 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.MetricNameTemplate; public class CaffeineMetricsRegistry { + public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.cache"; static final String CACHE_HITS = "cache-hits"; static final String CACHE_HITS_TOTAL = CACHE_HITS + "-total"; diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java index 6aa0e95c2..ceaf9736d 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java @@ -43,6 +43,7 @@ import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_LOAD_SUCCESS_TIME; import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_MISSES; import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_SIZE; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.METRIC_CONTEXT; /** * Records cache metrics managed by Caffeine {@code Cache#stats}. @@ -87,7 +88,7 @@ public CaffeineStatsCounter(final String groupName) { metrics = new org.apache.kafka.common.metrics.Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, - new KafkaMetricsContext("aiven.kafka.server.tieredstorage.cache") + new KafkaMetricsContext(METRIC_CONTEXT) ); metricsRegistry = new CaffeineMetricsRegistry(groupName); diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/Metrics.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/Metrics.java index 177d5885a..b0bc4b017 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/Metrics.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/Metrics.java @@ -35,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.METRIC_CONTEXT; import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.OBJECT_UPLOAD; import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.OBJECT_UPLOAD_BYTES; import static io.aiven.kafka.tieredstorage.metrics.MetricsRegistry.SEGMENT_COPY_TIME; @@ -69,7 +70,7 @@ public Metrics(final Time time, final MetricConfig metricConfig) { metricConfig, List.of(reporter), time, - new KafkaMetricsContext("aiven.kafka.server.tieredstorage") + new KafkaMetricsContext(METRIC_CONTEXT) ); metricsRegistry = new MetricsRegistry(); diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java index 87e4bf1b6..02320644f 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java @@ -25,7 +25,7 @@ import io.aiven.kafka.tieredstorage.ObjectKeyFactory; public class MetricsRegistry { - + public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage"; static final String METRIC_GROUP = "remote-storage-manager-metrics"; static final String TAG_NAME_OBJECT_TYPE = "object-type"; static final String[] OBJECT_TYPE_TAG_NAMES = {TAG_NAME_OBJECT_TYPE}; diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java index 57be10934..969d37bec 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java @@ -29,6 +29,7 @@ import org.apache.kafka.common.utils.Time; import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.ACTIVE_THREADS; +import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG; import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.PARALLELISM; import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.POOL_SIZE; import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.QUEUED_TASK_COUNT; @@ -51,7 +52,7 @@ public ThreadPoolMonitor(final String groupName, final ExecutorService pool) { final JmxReporter reporter = new JmxReporter(); metrics = new org.apache.kafka.common.metrics.Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, - new KafkaMetricsContext("aiven.kafka.server.tieredstorage.thread-pool") + new KafkaMetricsContext(METRIC_CONFIG) ); final var metricsRegistry = new ThreadPoolMonitorMetricsRegistry(groupName); registerSensor(metricsRegistry.activeThreadsTotalMetricName, ACTIVE_THREADS, this::activeThreadCount); diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java index 2ba9ba85b..589cbddd1 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.MetricNameTemplate; public class ThreadPoolMonitorMetricsRegistry { + public static final String METRIC_CONFIG = "aiven.kafka.server.tieredstorage.thread-pool"; static final String ACTIVE_THREADS = "active-thread-count"; private static final String ACTIVE_THREADS_TOTAL = ACTIVE_THREADS + "-total"; diff --git a/docs/build.gradle b/docs/build.gradle index 7ae589ca6..e2e3d1e43 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -31,3 +31,9 @@ tasks.register('genConfigDocs', JavaExec) { mainClass = 'io.aiven.kafka.tieredstorage.misc.ConfigDocs' standardOutput = new File("config.rst").newOutputStream() } + +tasks.register('genMetricsDocs', JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'io.aiven.kafka.tieredstorage.misc.MetricDocs' + standardOutput = new File("metrics.rst").newOutputStream() +} diff --git a/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricDocs.java b/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricDocs.java new file mode 100644 index 000000000..5294a5124 --- /dev/null +++ b/docs/src/main/java/io/aiven/kafka/tieredstorage/misc/MetricDocs.java @@ -0,0 +1,189 @@ +/* + * Copyright 2024 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.tieredstorage.misc; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.MetricNameTemplate; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.utils.Sanitizer; + +import io.aiven.kafka.tieredstorage.fetch.cache.ChunkCache; +import io.aiven.kafka.tieredstorage.fetch.index.MemorySegmentIndexesCache; +import io.aiven.kafka.tieredstorage.fetch.manifest.MemorySegmentManifestCache; +import io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry; +import io.aiven.kafka.tieredstorage.metrics.MetricsRegistry; +import io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry; + +public class MetricDocs { + public static void main(final String[] args) { + printSectionTitle("Core components metrics"); + System.out.println(); + printSubsectionTitle("RemoteStorageManager metrics"); + System.out.println(); + System.out.println(toRstTable(MetricsRegistry.METRIC_CONTEXT, new MetricsRegistry().all())); + + System.out.println(); + printSubsectionTitle("SegmentManifestCache metrics"); + System.out.println(); + System.out.println(toRstTable( + CaffeineMetricsRegistry.METRIC_CONTEXT, + new CaffeineMetricsRegistry(MemorySegmentManifestCache.METRIC_GROUP).all())); + System.out.println(); + System.out.println(toRstTable( + ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG, + new ThreadPoolMonitorMetricsRegistry(MemorySegmentManifestCache.THREAD_POOL_METRIC_GROUP).all())); + + System.out.println(); + printSubsectionTitle("SegmentIndexesCache metrics"); + System.out.println(toRstTable( + CaffeineMetricsRegistry.METRIC_CONTEXT, + new CaffeineMetricsRegistry(MemorySegmentIndexesCache.METRIC_GROUP).all())); + System.out.println(toRstTable( + ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG, + new ThreadPoolMonitorMetricsRegistry(MemorySegmentIndexesCache.THREAD_POOL_METRIC_GROUP).all())); + System.out.println(); + printSubsectionTitle("ChunkCache metrics"); + System.out.println(); + System.out.println(toRstTable( + CaffeineMetricsRegistry.METRIC_CONTEXT, + new CaffeineMetricsRegistry(ChunkCache.METRIC_GROUP).all())); + System.out.println(); + System.out.println(toRstTable( + ThreadPoolMonitorMetricsRegistry.METRIC_CONFIG, + new ThreadPoolMonitorMetricsRegistry(ChunkCache.THREAD_POOL_METRIC_GROUP).all())); + + System.out.println(); + printSectionTitle("Storage Backend metrics"); + System.out.println(); + printSubsectionTitle("AzureBlobStorage metrics"); + System.out.println(); + System.out.println(toRstTable( + io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.METRIC_CONTEXT, + new io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry().all())); + System.out.println(); + printSubsectionTitle("GcsStorage metrics"); + System.out.println(); + System.out.println(toRstTable( + io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT, + new io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry().all())); + System.out.println(); + printSubsectionTitle("S3Storage metrics"); + System.out.println(); + System.out.println(toRstTable( + io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.METRIC_CONTEXT, + new io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry().all())); + } + + public static String toRstTable(final String domain, final Iterable allMetrics) { + final Map> beansAndAttributes = new TreeMap<>(); + + try (final Metrics metrics = new Metrics()) { + for (final MetricNameTemplate template : allMetrics) { + final Map tags = new LinkedHashMap<>(); + for (final String s : template.tags()) { + tags.put(s, "{" + s + "}"); + } + + final MetricName metricName = metrics.metricName( + template.name(), + template.group(), + template.description(), + tags + ); + final String beanName = getMBeanName(domain, metricName); + beansAndAttributes.computeIfAbsent(beanName, k -> new TreeMap<>()); + final Map attrAndDesc = beansAndAttributes.get(beanName); + if (!attrAndDesc.containsKey(template.name())) { + attrAndDesc.put(template.name(), template.description()); + } else { + throw new IllegalArgumentException( + "mBean '" + beanName + + "' attribute '" + + template.name() + + "' is defined twice." + ); + } + } + } + + final StringBuilder b = new StringBuilder(); + + for (final Map.Entry> e : beansAndAttributes.entrySet()) { + // Add mBean name as a section title + b.append(e.getKey()).append("\n"); + b.append("=".repeat(e.getKey().length())).append("\n\n"); + + // Determine the maximum lengths for each column + final int maxAttrLength = Math.max("Attribute name".length(), + e.getValue().keySet().stream().mapToInt(String::length).max().orElse(0)); + final int maxDescLength = Math.max("Description".length(), + e.getValue().values().stream().mapToInt(String::length).max().orElse(0)); + + // Create the table header + final String headerFormat = "%-" + maxAttrLength + "s %-" + maxDescLength + "s\n"; + final String separatorLine = "=" + "=".repeat(maxAttrLength) + " " + "=".repeat(maxDescLength) + "\n"; + + b.append(separatorLine); + b.append(String.format(headerFormat, "Attribute name", "Description")); + b.append(separatorLine); + + // Add table rows + for (final Map.Entry e2 : e.getValue().entrySet()) { + b.append(String.format(headerFormat, e2.getKey(), e2.getValue())); + } + + // Close the table + b.append(separatorLine); + b.append("\n"); // Add an empty line between tables + } + + return b.toString(); + } + + static String getMBeanName(final String prefix, final MetricName metricName) { + final StringBuilder beanName = new StringBuilder(); + beanName.append(prefix); + beanName.append(":type="); + beanName.append(metricName.group()); + for (final Map.Entry entry : metricName.tags().entrySet()) { + if (entry.getKey().length() <= 0 || entry.getValue().length() <= 0) { + continue; + } + beanName.append(","); + beanName.append(entry.getKey()); + beanName.append("="); + beanName.append(Sanitizer.jmxSanitize(entry.getValue())); + } + return beanName.toString(); + } + + static void printSectionTitle(final String title) { + System.out.println("=================\n" + + title + "\n" + + "================="); + } + + static void printSubsectionTitle(final String title) { + System.out.println("-----------------\n" + + title + "\n" + + "-----------------"); + } +} diff --git a/metrics.rst b/metrics.rst new file mode 100644 index 000000000..395a4092f --- /dev/null +++ b/metrics.rst @@ -0,0 +1,367 @@ +================= +Core components metrics +================= + +----------------- +RemoteStorageManager metrics +----------------- + +aiven.kafka.server.tieredstorage:type=remote-storage-manager-metrics +==================================================================== + +==================================== =========== +Attribute name Description +==================================== =========== +object-upload-bytes-rate +object-upload-bytes-total +object-upload-rate +object-upload-total +segment-copy-time-avg +segment-copy-time-max +segment-delete-bytes-total +segment-delete-errors-rate +segment-delete-errors-total +segment-delete-rate +segment-delete-time-avg +segment-delete-time-max +segment-delete-total +segment-fetch-requested-bytes-rate +segment-fetch-requested-bytes-total +==================================== =========== + +aiven.kafka.server.tieredstorage:type=remote-storage-manager-metrics,object-type="{object-type}" +================================================================================================ + +========================== =========== +Attribute name Description +========================== =========== +object-upload-bytes-rate +object-upload-bytes-total +object-upload-rate +object-upload-total +========================== =========== + +aiven.kafka.server.tieredstorage:type=remote-storage-manager-metrics,topic="{topic}" +==================================================================================== + +==================================== =========== +Attribute name Description +==================================== =========== +object-upload-bytes-rate +object-upload-bytes-total +object-upload-rate +object-upload-total +segment-copy-time-avg +segment-copy-time-max +segment-delete-bytes-total +segment-delete-errors-rate +segment-delete-errors-total +segment-delete-rate +segment-delete-time-avg +segment-delete-time-max +segment-delete-total +segment-fetch-requested-bytes-rate +segment-fetch-requested-bytes-total +==================================== =========== + +aiven.kafka.server.tieredstorage:type=remote-storage-manager-metrics,topic="{topic}",object-type="{object-type}" +================================================================================================================ + +========================== =========== +Attribute name Description +========================== =========== +object-upload-bytes-rate +object-upload-bytes-total +object-upload-rate +object-upload-total +========================== =========== + +aiven.kafka.server.tieredstorage:type=remote-storage-manager-metrics,topic="{topic}",partition="{partition}" +============================================================================================================ + +==================================== =========== +Attribute name Description +==================================== =========== +object-upload-bytes-rate +object-upload-bytes-total +object-upload-rate +object-upload-total +segment-copy-time-avg +segment-copy-time-max +segment-delete-bytes-total +segment-delete-errors-rate +segment-delete-errors-total +segment-delete-rate +segment-delete-time-avg +segment-delete-time-max +segment-delete-total +segment-fetch-requested-bytes-rate +segment-fetch-requested-bytes-total +==================================== =========== + +aiven.kafka.server.tieredstorage:type=remote-storage-manager-metrics,topic="{topic}",partition="{partition}",object-type="{object-type}" +======================================================================================================================================== + +========================== =========== +Attribute name Description +========================== =========== +object-upload-bytes-rate +object-upload-bytes-total +object-upload-rate +object-upload-total +========================== =========== + + + +----------------- +SegmentManifestCache metrics +----------------- + +aiven.kafka.server.tieredstorage.cache:type=segment-manifest-cache-metrics +========================================================================== + +============================== =========== +Attribute name Description +============================== =========== +cache-eviction-total +cache-eviction-weight-total +cache-hits-total +cache-load-failure-time-total +cache-load-failure-total +cache-load-success-time-total +cache-load-success-total +cache-misses-total +cache-size-total +============================== =========== + +aiven.kafka.server.tieredstorage.cache:type=segment-manifest-cache-metrics,cause="{cause}" +========================================================================================== + +============================ =========== +Attribute name Description +============================ =========== +cache-eviction-total +cache-eviction-weight-total +============================ =========== + + + +aiven.kafka.server.tieredstorage.thread-pool:type=segment-manifest-cache-thread-pool-metrics +============================================================================================ + +=========================== =========== +Attribute name Description +=========================== =========== +active-thread-count-total +parallelism-total +pool-size-total +queued-task-count-total +running-thread-count-total +steal-task-count-total +=========================== =========== + + + +----------------- +SegmentIndexesCache metrics +----------------- +aiven.kafka.server.tieredstorage.cache:type=segment-indexes-cache-metrics +========================================================================= + +============================== =========== +Attribute name Description +============================== =========== +cache-eviction-total +cache-eviction-weight-total +cache-hits-total +cache-load-failure-time-total +cache-load-failure-total +cache-load-success-time-total +cache-load-success-total +cache-misses-total +cache-size-total +============================== =========== + +aiven.kafka.server.tieredstorage.cache:type=segment-indexes-cache-metrics,cause="{cause}" +========================================================================================= + +============================ =========== +Attribute name Description +============================ =========== +cache-eviction-total +cache-eviction-weight-total +============================ =========== + + +aiven.kafka.server.tieredstorage.thread-pool:type=segment-indexes-cache-thread-pool-metrics +=========================================================================================== + +=========================== =========== +Attribute name Description +=========================== =========== +active-thread-count-total +parallelism-total +pool-size-total +queued-task-count-total +running-thread-count-total +steal-task-count-total +=========================== =========== + + + +----------------- +ChunkCache metrics +----------------- + +aiven.kafka.server.tieredstorage.cache:type=chunk-cache-metrics +=============================================================== + +============================== =========== +Attribute name Description +============================== =========== +cache-eviction-total +cache-eviction-weight-total +cache-hits-total +cache-load-failure-time-total +cache-load-failure-total +cache-load-success-time-total +cache-load-success-total +cache-misses-total +cache-size-total +============================== =========== + +aiven.kafka.server.tieredstorage.cache:type=chunk-cache-metrics,cause="{cause}" +=============================================================================== + +============================ =========== +Attribute name Description +============================ =========== +cache-eviction-total +cache-eviction-weight-total +============================ =========== + + + +aiven.kafka.server.tieredstorage.thread-pool:type=chunk-cache-thread-pool-metrics +================================================================================= + +=========================== =========== +Attribute name Description +=========================== =========== +active-thread-count-total +parallelism-total +pool-size-total +queued-task-count-total +running-thread-count-total +steal-task-count-total +=========================== =========== + + + +================= +Storage Backend metrics +================= + +----------------- +AzureBlobStorage metrics +----------------- + +aiven.kafka.server.tieredstorage.azure:type=azure-blob-storage-client-metrics +============================================================================= + +======================== =========== +Attribute name Description +======================== =========== +blob-delete-rate +blob-delete-total +blob-get-rate +blob-get-total +blob-upload-rate +blob-upload-total +block-list-upload-rate +block-list-upload-total +block-upload-rate +block-upload-total +======================== =========== + + + +----------------- +GcsStorage metrics +----------------- + +aiven.kafka.server.tieredstorage.gcs:type=gcs-client-metrics +============================================================ + +================================ =========== +Attribute name Description +================================ =========== +object-delete-rate +object-delete-total +object-get-rate +object-get-total +object-metadata-get-rate +object-metadata-get-total +resumable-chunk-upload-rate +resumable-chunk-upload-total +resumable-upload-initiate-rate +resumable-upload-initiate-total +================================ =========== + + + +----------------- +S3Storage metrics +----------------- + +aiven.kafka.server.tieredstorage.s3:type=s3-client-metrics +========================================================== + +========================================= =========== +Attribute name Description +========================================= =========== +abort-multipart-upload-requests-rate +abort-multipart-upload-requests-total +abort-multipart-upload-time-avg +abort-multipart-upload-time-max +complete-multipart-upload-requests-rate +complete-multipart-upload-requests-total +complete-multipart-upload-time-avg +complete-multipart-upload-time-max +configured-timeout-errors-rate +configured-timeout-errors-total +create-multipart-upload-requests-rate +create-multipart-upload-requests-total +create-multipart-upload-time-avg +create-multipart-upload-time-max +delete-object-requests-rate +delete-object-requests-total +delete-object-time-avg +delete-object-time-max +delete-objects-requests-rate +delete-objects-requests-total +delete-objects-time-avg +delete-objects-time-max +get-object-requests-rate +get-object-requests-total +get-object-time-avg +get-object-time-max +io-errors-rate +io-errors-total +other-errors-rate +other-errors-total +put-object-requests-rate +put-object-requests-total +put-object-time-avg +put-object-time-max +server-errors-rate +server-errors-total +throttling-errors-rate +throttling-errors-total +upload-part-requests-rate +upload-part-requests-total +upload-part-time-avg +upload-part-time-max +========================================= =========== + + diff --git a/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java index 2ed33fcd3..75881989e 100644 --- a/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java +++ b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java @@ -53,6 +53,7 @@ import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD; import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD_RATE_METRIC_NAME; import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.METRIC_CONTEXT; public class MetricCollector { @@ -66,7 +67,7 @@ public MetricCollector(final AzureBlobStorageConfig config) { final Metrics metrics = new Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, - new KafkaMetricsContext("aiven.kafka.server.tieredstorage.azure") + new KafkaMetricsContext(METRIC_CONTEXT) ); policy = new MetricsPolicy(metrics, pathPattern()); } diff --git a/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java index fdba497c5..c0bdcd145 100644 --- a/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java +++ b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.MetricNameTemplate; public class MetricRegistry { + public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.azure"; static final String METRIC_GROUP = "azure-blob-storage-client-metrics"; static final String BLOB_DELETE = "blob-delete"; static final String BLOB_DELETE_RATE = BLOB_DELETE + "-rate"; diff --git a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java index b57825bbd..a85317bcb 100644 --- a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java +++ b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java @@ -38,6 +38,7 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.http.HttpTransportOptions; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT; import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE; import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_RATE_METRIC_NAME; import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_TOTAL_METRIC_NAME; @@ -92,7 +93,7 @@ public MetricCollector() { metrics = new org.apache.kafka.common.metrics.Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, - new KafkaMetricsContext("aiven.kafka.server.tieredstorage.gcs") + new KafkaMetricsContext(METRIC_CONTEXT) ); getObjectMetadataRequests = createSensor( diff --git a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java index 1028a1557..b773e8669 100644 --- a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java +++ b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.MetricNameTemplate; public class MetricRegistry { + public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.gcs"; static final String METRIC_GROUP = "gcs-client-metrics"; static final String OBJECT_METADATA_GET = "object-metadata-get"; static final String OBJECT_METADATA_GET_RATE = OBJECT_METADATA_GET + "-rate"; diff --git a/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java index 4a45b4714..db882198f 100644 --- a/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java +++ b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java @@ -82,6 +82,7 @@ import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS; import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS_RATE_METRIC_NAME; import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.METRIC_CONTEXT; import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS; import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS_RATE_METRIC_NAME; import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS_TOTAL_METRIC_NAME; @@ -123,7 +124,7 @@ public MetricCollector() { metrics = new org.apache.kafka.common.metrics.Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, - new KafkaMetricsContext("aiven.kafka.server.tieredstorage.s3") + new KafkaMetricsContext(METRIC_CONTEXT) ); final Sensor getObjectRequestsSensor = createRequestsSensor( GET_OBJECT_REQUESTS, diff --git a/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java index 98b8e843c..dbbdeeaf8 100644 --- a/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java +++ b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.MetricNameTemplate; public class MetricRegistry { + public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.s3"; static final String METRIC_GROUP = "s3-client-metrics"; static final String GET_OBJECT_REQUESTS = "get-object-requests"; static final String GET_OBJECT_REQUESTS_RATE = GET_OBJECT_REQUESTS + "-rate";