From daec916e9577a5d527272cfff3090f3bb5006525 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Mon, 14 Oct 2024 18:40:44 +0300 Subject: [PATCH] refactor: add metrics registry for all components Most components had metric names hardcoded. Adding a metrics registry for each component to contain all metric name templates, and export them for document generation. --- .../metrics/CaffeineMetricsRegistry.java | 102 ++++++++ .../metrics/CaffeineStatsCounter.java | 124 +++++---- .../metrics/MetricsRegistry.java | 67 +++++ .../metrics/ThreadPoolMonitor.java | 39 ++- .../ThreadPoolMonitorMetricsRegistry.java | 66 +++++ .../storage/azure/MetricCollector.java | 67 +++-- .../storage/azure/MetricRegistry.java | 75 ++++++ .../storage/gcs/MetricCollector.java | 63 ++++- .../storage/gcs/MetricRegistry.java | 76 ++++++ .../storage/s3/MetricCollector.java | 238 +++++++++++++++--- .../storage/s3/MetricRegistry.java | 222 ++++++++++++++++ 11 files changed, 1010 insertions(+), 129 deletions(-) create mode 100644 core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java create mode 100644 core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java create mode 100644 storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java create mode 100644 storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java create mode 100644 storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java new file mode 100644 index 000000000..ff2782b01 --- /dev/null +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineMetricsRegistry.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.tieredstorage.metrics; + +import java.util.List; + +import org.apache.kafka.common.MetricNameTemplate; + +public class CaffeineMetricsRegistry { + + static final String CACHE_HITS = "cache-hits"; + static final String CACHE_HITS_TOTAL = CACHE_HITS + "-total"; + static final String CACHE_MISSES = "cache-misses"; + static final String CACHE_MISSES_TOTAL = CACHE_MISSES + "-total"; + static final String CACHE_LOAD = "cache-load"; + static final String CACHE_LOAD_SUCCESS = CACHE_LOAD + "-success"; + static final String CACHE_LOAD_SUCCESS_TOTAL = CACHE_LOAD_SUCCESS + "-total"; + static final String CACHE_LOAD_SUCCESS_TIME = CACHE_LOAD_SUCCESS + "-time"; + static final String CACHE_LOAD_SUCCESS_TIME_TOTAL = CACHE_LOAD_SUCCESS_TIME + "-total"; + static final String CACHE_LOAD_FAILURE = CACHE_LOAD + "-failure"; + static final String CACHE_LOAD_FAILURE_TOTAL = CACHE_LOAD_FAILURE + "-total"; + static final String CACHE_LOAD_FAILURE_TIME = CACHE_LOAD_FAILURE + "-time"; + static final String CACHE_LOAD_FAILURE_TIME_TOTAL = CACHE_LOAD_FAILURE_TIME + "-total"; + + static final String CACHE_EVICTION = "cache-eviction"; + static final String CACHE_EVICTION_TOTAL = CACHE_EVICTION + "-total"; + static final String CACHE_EVICTION_WEIGHT = CACHE_EVICTION + "-weight"; + static final String CACHE_EVICTION_WEIGHT_TOTAL = CACHE_EVICTION_WEIGHT + "-total"; + + static final String CACHE_SIZE = "cache-size"; + static final String CACHE_SIZE_TOTAL = CACHE_SIZE + "-total"; + + final String groupName; + + final MetricNameTemplate cacheHitsMetricName; + final MetricNameTemplate cacheMissesMetricName; + final MetricNameTemplate cacheLoadSuccessMetricName; + final MetricNameTemplate cacheLoadSuccessTimeMetricName; + final MetricNameTemplate cacheLoadFailureMetricName; + final MetricNameTemplate cacheLoadFailureTimeMetricName; + final MetricNameTemplate cacheEvictionMetricName; + final MetricNameTemplate cacheEvictionByCauseMetricName; + final MetricNameTemplate cacheEvictionWeightMetricName; + final MetricNameTemplate cacheEvictionWeightByCauseMetricName; + final MetricNameTemplate cacheSizeTotalMetricName; + + public CaffeineMetricsRegistry(final String groupName) { + this.groupName = groupName; + cacheHitsMetricName = + new MetricNameTemplate(CACHE_HITS_TOTAL, groupName, ""); + cacheMissesMetricName = + new MetricNameTemplate(CACHE_MISSES_TOTAL, groupName, ""); + cacheLoadSuccessMetricName = + new MetricNameTemplate(CACHE_LOAD_SUCCESS_TOTAL, groupName, ""); + cacheLoadSuccessTimeMetricName = + new MetricNameTemplate(CACHE_LOAD_SUCCESS_TIME_TOTAL, groupName, ""); + cacheLoadFailureMetricName = + new MetricNameTemplate(CACHE_LOAD_FAILURE_TOTAL, groupName, ""); + cacheLoadFailureTimeMetricName = + new MetricNameTemplate(CACHE_LOAD_FAILURE_TIME_TOTAL, groupName, ""); + cacheEvictionMetricName = + new MetricNameTemplate(CACHE_EVICTION_TOTAL, groupName, ""); + cacheEvictionByCauseMetricName = + new MetricNameTemplate(CACHE_EVICTION_TOTAL, groupName, "", "cause"); + cacheEvictionWeightMetricName = + new MetricNameTemplate(CACHE_EVICTION_WEIGHT_TOTAL, groupName, ""); + cacheEvictionWeightByCauseMetricName = + new MetricNameTemplate(CACHE_EVICTION_WEIGHT_TOTAL, groupName, "", "cause"); + cacheSizeTotalMetricName = + new MetricNameTemplate(CACHE_SIZE_TOTAL, groupName, ""); + } + + public List all() { + return List.of( + cacheHitsMetricName, + cacheMissesMetricName, + cacheLoadSuccessMetricName, + cacheLoadSuccessTimeMetricName, + cacheLoadFailureMetricName, + cacheLoadFailureTimeMetricName, + cacheEvictionMetricName, + cacheEvictionByCauseMetricName, + cacheEvictionWeightMetricName, + cacheEvictionWeightByCauseMetricName, + cacheSizeTotalMetricName + ); + } +} diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java index 742dca5e5..6aa0e95c2 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/CaffeineStatsCounter.java @@ -34,6 +34,16 @@ import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.github.benmanes.caffeine.cache.stats.StatsCounter; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_EVICTION; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_EVICTION_WEIGHT; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_HITS; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_LOAD_FAILURE; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_LOAD_FAILURE_TIME; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_LOAD_SUCCESS; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_LOAD_SUCCESS_TIME; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_MISSES; +import static io.aiven.kafka.tieredstorage.metrics.CaffeineMetricsRegistry.CACHE_SIZE; + /** * Records cache metrics managed by Caffeine {@code Cache#stats}. * @@ -43,29 +53,6 @@ * Micrometer */ public class CaffeineStatsCounter implements StatsCounter { - - static final String CACHE_HITS = "cache-hits"; - static final String CACHE_HITS_TOTAL = CACHE_HITS + "-total"; - static final String CACHE_MISSES = "cache-misses"; - static final String CACHE_MISSES_TOTAL = CACHE_MISSES + "-total"; - static final String CACHE_LOAD = "cache-load"; - static final String CACHE_LOAD_SUCCESS = CACHE_LOAD + "-success"; - static final String CACHE_LOAD_SUCCESS_TOTAL = CACHE_LOAD_SUCCESS + "-total"; - static final String CACHE_LOAD_SUCCESS_TIME = CACHE_LOAD_SUCCESS + "-time"; - static final String CACHE_LOAD_SUCCESS_TIME_TOTAL = CACHE_LOAD_SUCCESS_TIME + "-total"; - static final String CACHE_LOAD_FAILURE = CACHE_LOAD + "-failure"; - static final String CACHE_LOAD_FAILURE_TOTAL = CACHE_LOAD_FAILURE + "-total"; - static final String CACHE_LOAD_FAILURE_TIME = CACHE_LOAD_FAILURE + "-time"; - static final String CACHE_LOAD_FAILURE_TIME_TOTAL = CACHE_LOAD_FAILURE_TIME + "-total"; - - static final String CACHE_EVICTION = "cache-eviction"; - static final String CACHE_EVICTION_TOTAL = CACHE_EVICTION + "-total"; - static final String CACHE_EVICTION_WEIGHT = CACHE_EVICTION + "-weight"; - static final String CACHE_EVICTION_WEIGHT_TOTAL = CACHE_EVICTION_WEIGHT + "-total"; - - static final String CACHE_SIZE = "cache-size"; - static final String CACHE_SIZE_TOTAL = CACHE_SIZE + "-total"; - private final org.apache.kafka.common.metrics.Metrics metrics; private final LongAdder cacheHitCount; @@ -78,7 +65,7 @@ public class CaffeineStatsCounter implements StatsCounter { private final LongAdder cacheEvictionWeightTotal; private final ConcurrentHashMap cacheEvictionCountByCause; private final ConcurrentHashMap cacheEvictionWeightByCause; - private final String groupName; + private final CaffeineMetricsRegistry metricsRegistry; public CaffeineStatsCounter(final String groupName) { cacheHitCount = new LongAdder(); @@ -90,8 +77,6 @@ public CaffeineStatsCounter(final String groupName) { cacheEvictionCountTotal = new LongAdder(); cacheEvictionWeightTotal = new LongAdder(); - this.groupName = groupName; - cacheEvictionCountByCause = new ConcurrentHashMap<>(); Arrays.stream(RemovalCause.values()).forEach(cause -> cacheEvictionCountByCause.put(cause, new LongAdder())); @@ -105,39 +90,81 @@ public CaffeineStatsCounter(final String groupName) { new KafkaMetricsContext("aiven.kafka.server.tieredstorage.cache") ); - initSensor(CACHE_HITS, CACHE_HITS_TOTAL, cacheHitCount); - initSensor(CACHE_MISSES, CACHE_MISSES_TOTAL, cacheMissCount); - initSensor(CACHE_LOAD_SUCCESS, CACHE_LOAD_SUCCESS_TOTAL, cacheLoadSuccessCount); - initSensor(CACHE_LOAD_SUCCESS_TIME, CACHE_LOAD_SUCCESS_TIME_TOTAL, cacheLoadSuccessTimeTotal); - initSensor(CACHE_LOAD_FAILURE, CACHE_LOAD_FAILURE_TOTAL, cacheLoadFailureCount); - initSensor(CACHE_LOAD_FAILURE_TIME, CACHE_LOAD_FAILURE_TIME_TOTAL, cacheLoadFailureTimeTotal); - initSensor(CACHE_EVICTION, CACHE_EVICTION_TOTAL, cacheEvictionCountTotal); + metricsRegistry = new CaffeineMetricsRegistry(groupName); + initSensor( + metricsRegistry.cacheHitsMetricName, + CACHE_HITS, + cacheHitCount + ); + initSensor( + metricsRegistry.cacheMissesMetricName, + CACHE_MISSES, + cacheMissCount + ); + initSensor( + metricsRegistry.cacheLoadSuccessMetricName, + CACHE_LOAD_SUCCESS, + cacheLoadSuccessCount + ); + initSensor( + metricsRegistry.cacheLoadSuccessTimeMetricName, + CACHE_LOAD_SUCCESS_TIME, + cacheLoadSuccessTimeTotal + ); + initSensor( + metricsRegistry.cacheLoadFailureMetricName, + CACHE_LOAD_FAILURE, + cacheLoadFailureCount + ); + initSensor( + metricsRegistry.cacheLoadFailureTimeMetricName, + CACHE_LOAD_FAILURE_TIME, + cacheLoadFailureTimeTotal + ); + initSensor( + metricsRegistry.cacheEvictionMetricName, + CACHE_EVICTION, + cacheEvictionCountTotal + ); Arrays.stream(RemovalCause.values()).forEach(cause -> - initSensor("cause." + cause.name() + "." + CACHE_EVICTION, CACHE_EVICTION_TOTAL, - cacheEvictionCountByCause.get(cause), () -> Map.of("cause", cause.name()), "cause") + initSensor( + metricsRegistry.cacheEvictionByCauseMetricName, + "cause." + cause.name() + "." + CACHE_EVICTION, + cacheEvictionCountByCause.get(cause), + () -> Map.of("cause", cause.name()) + ) ); - initSensor(CACHE_EVICTION_WEIGHT, CACHE_EVICTION_WEIGHT_TOTAL, cacheEvictionWeightTotal); + initSensor( + metricsRegistry.cacheEvictionWeightMetricName, + CACHE_EVICTION_WEIGHT, + cacheEvictionWeightTotal + ); Arrays.stream(RemovalCause.values()).forEach(cause -> - initSensor("cause." + cause.name() + "." + CACHE_EVICTION, CACHE_EVICTION_WEIGHT_TOTAL, - cacheEvictionWeightByCause.get(cause), () -> Map.of("cause", cause.name()), "cause") + initSensor( + metricsRegistry.cacheEvictionWeightByCauseMetricName, + "cause." + cause.name() + "." + CACHE_EVICTION, + cacheEvictionWeightByCause.get(cause), + () -> Map.of("cause", cause.name()) + ) ); } - private void initSensor(final String sensorName, - final String metricName, - final LongAdder value, - final Supplier> tagsSupplier, - final String... tagNames) { - final var name = new MetricNameTemplate(metricName, groupName, "", tagNames); + private void initSensor( + final MetricNameTemplate metricNameTemplate, + final String sensorName, + final LongAdder value, + final Supplier> tagsSupplier + ) { new SensorProvider(metrics, sensorName, tagsSupplier) - .with(name, new MeasurableValue(value::sum)) + .with(metricNameTemplate, new MeasurableValue(value::sum)) .get(); } - private void initSensor(final String sensorName, final String metricName, final LongAdder value) { - initSensor(sensorName, metricName, value, Collections::emptyMap); + private void initSensor(final MetricNameTemplate metricNameTemplate, final String sensorName, + final LongAdder value) { + initSensor(metricNameTemplate, sensorName, value, Collections::emptyMap); } @Override @@ -193,9 +220,8 @@ public void recordHit() { * @param sizeSupplier operation from cache to provide cache size value */ public void registerSizeMetric(final Supplier sizeSupplier) { - final var name = new MetricNameTemplate(CACHE_SIZE_TOTAL, groupName, ""); new SensorProvider(metrics, CACHE_SIZE) - .with(name, new MeasurableValue(sizeSupplier)) + .with(metricsRegistry.cacheSizeTotalMetricName, new MeasurableValue(sizeSupplier)) .get(); } diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java index ee3265f6b..87e4bf1b6 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/MetricsRegistry.java @@ -16,6 +16,7 @@ package io.aiven.kafka.tieredstorage.metrics; +import java.util.List; import java.util.Map; import org.apache.kafka.common.MetricNameTemplate; @@ -252,4 +253,70 @@ static Map topicPartitionAndObjectTypeTags(final TopicPartition static Map objectTypeTags(final ObjectKeyFactory.Suffix suffix) { return Map.of(TAG_NAME_OBJECT_TYPE, suffix.value); } + + public Iterable all() { + return List.of( + // segment copy + segmentCopyTimeAvg, + segmentCopyTimeAvgByTopic, + segmentCopyTimeAvgByTopicPartition, + segmentCopyTimeMax, + segmentCopyTimeMaxByTopic, + segmentCopyTimeMaxByTopicPartition, + // segment delete + segmentDeleteRequestsRate, + segmentDeleteRequestsRateByTopic, + segmentDeleteRequestsRateByTopicPartition, + segmentDeleteRequestsTotal, + segmentDeleteRequestsTotalByTopic, + segmentDeleteRequestsTotalByTopicPartition, + segmentDeleteBytesTotal, + segmentDeleteBytesTotalByTopic, + segmentDeleteBytesTotalByTopicPartition, + segmentDeleteTimeAvg, + segmentDeleteTimeAvgByTopic, + segmentDeleteTimeAvgByTopicPartition, + segmentDeleteTimeMax, + segmentDeleteTimeMaxByTopic, + segmentDeleteTimeMaxByTopicPartition, + segmentDeleteErrorsRate, + segmentDeleteErrorsRateByTopic, + segmentDeleteErrorsRateByTopicPartition, + segmentDeleteErrorsTotal, + segmentDeleteErrorsTotalByTopic, + segmentDeleteErrorsTotalByTopicPartition, + // segment fetch + segmentFetchRequestedBytesRate, + segmentFetchRequestedBytesRateByTopic, + segmentFetchRequestedBytesRateByTopicPartition, + segmentFetchRequestedBytesTotal, + segmentFetchRequestedBytesTotalByTopic, + segmentFetchRequestedBytesTotalByTopicPartition, + // object upload + objectUploadRequestsRate, + objectUploadRequestsRateByTopic, + objectUploadRequestsRateByTopicPartition, + objectUploadRequestsRateByObjectType, + objectUploadRequestsRateByTopicAndObjectType, + objectUploadRequestsRateByTopicPartitionAndObjectType, + objectUploadRequestsTotal, + objectUploadRequestsTotalByTopic, + objectUploadRequestsTotalByTopicPartition, + objectUploadRequestsTotalByObjectType, + objectUploadRequestsTotalByTopicAndObjectType, + objectUploadRequestsTotalByTopicPartitionAndObjectType, + objectUploadBytesRate, + objectUploadBytesRateByTopic, + objectUploadBytesRateByTopicPartition, + objectUploadBytesRateByObjectType, + objectUploadBytesRateByTopicAndObjectType, + objectUploadBytesRateByTopicPartitionAndObjectType, + objectUploadBytesTotal, + objectUploadBytesTotalByTopic, + objectUploadBytesTotalByTopicPartition, + objectUploadBytesTotalByObjectType, + objectUploadBytesTotalByTopicAndObjectType, + objectUploadBytesTotalByTopicPartitionAndObjectType + ); + } } diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java index 8684a8941..57be10934 100644 --- a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitor.java @@ -28,25 +28,19 @@ import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.Time; +import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.ACTIVE_THREADS; +import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.PARALLELISM; +import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.POOL_SIZE; +import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.QUEUED_TASK_COUNT; +import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.RUNNING_THREADS; +import static io.aiven.kafka.tieredstorage.metrics.ThreadPoolMonitorMetricsRegistry.STEAL_TASK_COUNT; + public class ThreadPoolMonitor { // only fork-join pool is supported; but could be extended to other fixed-sized pools final ForkJoinPool pool; private final Metrics metrics; String groupName; - private static final String ACTIVE_THREADS = "active-thread-count"; - private static final String ACTIVE_THREADS_TOTAL = ACTIVE_THREADS + "-total"; - private static final String RUNNING_THREADS = "running-thread-count"; - private static final String RUNNING_THREADS_TOTAL = RUNNING_THREADS + "-total"; - private static final String POOL_SIZE = "pool-size"; - private static final String POOL_SIZE_TOTAL = POOL_SIZE + "-total"; - private static final String PARALLELISM = "parallelism"; - private static final String PARALLELISM_TOTAL = PARALLELISM + "-total"; - private static final String QUEUED_TASK_COUNT = "queued-task-count"; - private static final String QUEUED_TASK_COUNT_TOTAL = QUEUED_TASK_COUNT + "-total"; - private static final String STEAL_TASK_COUNT = "steal-task-count"; - private static final String STEAL_TASK_COUNT_TOTAL = STEAL_TASK_COUNT + "-total"; - public ThreadPoolMonitor(final String groupName, final ExecutorService pool) { this.groupName = groupName; if (!(pool instanceof ForkJoinPool)) { @@ -59,19 +53,18 @@ public ThreadPoolMonitor(final String groupName, final ExecutorService pool) { new MetricConfig(), List.of(reporter), Time.SYSTEM, new KafkaMetricsContext("aiven.kafka.server.tieredstorage.thread-pool") ); - - registerSensor(ACTIVE_THREADS_TOTAL, ACTIVE_THREADS, this::activeThreadCount); - registerSensor(RUNNING_THREADS_TOTAL, RUNNING_THREADS, this::runningThreadCount); - registerSensor(POOL_SIZE_TOTAL, POOL_SIZE, this::poolSize); - registerSensor(PARALLELISM_TOTAL, PARALLELISM, this::parallelism); - registerSensor(QUEUED_TASK_COUNT_TOTAL, QUEUED_TASK_COUNT, this::queuedTaskCount); - registerSensor(STEAL_TASK_COUNT_TOTAL, STEAL_TASK_COUNT, this::stealTaskCount); + final var metricsRegistry = new ThreadPoolMonitorMetricsRegistry(groupName); + registerSensor(metricsRegistry.activeThreadsTotalMetricName, ACTIVE_THREADS, this::activeThreadCount); + registerSensor(metricsRegistry.runningThreadsTotalMetricName, RUNNING_THREADS, this::runningThreadCount); + registerSensor(metricsRegistry.poolSizeTotalMetricName, POOL_SIZE, this::poolSize); + registerSensor(metricsRegistry.parallelismTotalMetricName, PARALLELISM, this::parallelism); + registerSensor(metricsRegistry.queuedTaskCountTotalMetricName, QUEUED_TASK_COUNT, this::queuedTaskCount); + registerSensor(metricsRegistry.stealTaskCountTotalMetricName, STEAL_TASK_COUNT, this::stealTaskCount); } - void registerSensor(final String metricName, final String sensorName, final Supplier supplier) { - final var name = new MetricNameTemplate(metricName, groupName, ""); + void registerSensor(final MetricNameTemplate metricName, final String sensorName, final Supplier supplier) { new SensorProvider(metrics, sensorName) - .with(name, new MeasurableValue(supplier)) + .with(metricName, new MeasurableValue(supplier)) .get(); } diff --git a/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java new file mode 100644 index 000000000..2ba9ba85b --- /dev/null +++ b/core/src/main/java/io/aiven/kafka/tieredstorage/metrics/ThreadPoolMonitorMetricsRegistry.java @@ -0,0 +1,66 @@ +/* + * Copyright 2024 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.tieredstorage.metrics; + +import java.util.List; + +import org.apache.kafka.common.MetricNameTemplate; + +public class ThreadPoolMonitorMetricsRegistry { + + static final String ACTIVE_THREADS = "active-thread-count"; + private static final String ACTIVE_THREADS_TOTAL = ACTIVE_THREADS + "-total"; + static final String RUNNING_THREADS = "running-thread-count"; + private static final String RUNNING_THREADS_TOTAL = RUNNING_THREADS + "-total"; + static final String POOL_SIZE = "pool-size"; + private static final String POOL_SIZE_TOTAL = POOL_SIZE + "-total"; + static final String PARALLELISM = "parallelism"; + private static final String PARALLELISM_TOTAL = PARALLELISM + "-total"; + static final String QUEUED_TASK_COUNT = "queued-task-count"; + private static final String QUEUED_TASK_COUNT_TOTAL = QUEUED_TASK_COUNT + "-total"; + static final String STEAL_TASK_COUNT = "steal-task-count"; + private static final String STEAL_TASK_COUNT_TOTAL = STEAL_TASK_COUNT + "-total"; + + final String groupName; + final MetricNameTemplate activeThreadsTotalMetricName; + final MetricNameTemplate runningThreadsTotalMetricName; + final MetricNameTemplate poolSizeTotalMetricName; + final MetricNameTemplate parallelismTotalMetricName; + final MetricNameTemplate queuedTaskCountTotalMetricName; + final MetricNameTemplate stealTaskCountTotalMetricName; + + public ThreadPoolMonitorMetricsRegistry(final String groupName) { + this.groupName = groupName; + activeThreadsTotalMetricName = new MetricNameTemplate(ACTIVE_THREADS_TOTAL, groupName, ""); + runningThreadsTotalMetricName = new MetricNameTemplate(RUNNING_THREADS_TOTAL, groupName, ""); + poolSizeTotalMetricName = new MetricNameTemplate(POOL_SIZE_TOTAL, groupName, ""); + parallelismTotalMetricName = new MetricNameTemplate(PARALLELISM_TOTAL, groupName, ""); + queuedTaskCountTotalMetricName = new MetricNameTemplate(QUEUED_TASK_COUNT_TOTAL, groupName, ""); + stealTaskCountTotalMetricName = new MetricNameTemplate(STEAL_TASK_COUNT_TOTAL, groupName, ""); + } + + public List all() { + return List.of( + activeThreadsTotalMetricName, + runningThreadsTotalMetricName, + poolSizeTotalMetricName, + parallelismTotalMetricName, + queuedTaskCountTotalMetricName, + stealTaskCountTotalMetricName + ); + } +} diff --git a/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java index 000c903a8..2ed33fcd3 100644 --- a/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java +++ b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java @@ -38,22 +38,37 @@ import com.azure.core.http.policy.HttpPipelinePolicy; import reactor.core.publisher.Mono; -public class MetricCollector { - private final org.apache.kafka.common.metrics.Metrics metrics; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_DELETE; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_DELETE_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_DELETE_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_GET; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_GET_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_GET_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_UPLOAD; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_UPLOAD_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_UPLOAD_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_LIST_UPLOAD; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_LIST_UPLOAD_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD_TOTAL_METRIC_NAME; - private static final String METRIC_GROUP = "azure-blob-storage-client-metrics"; +public class MetricCollector { final AzureBlobStorageConfig config; + final MetricsPolicy policy; - MetricCollector(final AzureBlobStorageConfig config) { + public MetricCollector(final AzureBlobStorageConfig config) { this.config = config; final JmxReporter reporter = new JmxReporter(); - metrics = new org.apache.kafka.common.metrics.Metrics( + final Metrics metrics = new Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, new KafkaMetricsContext("aiven.kafka.server.tieredstorage.azure") ); + policy = new MetricsPolicy(metrics, pathPattern()); } Pattern pathPattern() { @@ -64,7 +79,7 @@ Pattern pathPattern() { } MetricsPolicy policy() { - return new MetricsPolicy(metrics, pathPattern()); + return policy; } static class MetricsPolicy implements HttpPipelinePolicy { @@ -83,17 +98,41 @@ static class MetricsPolicy implements HttpPipelinePolicy { MetricsPolicy(final Metrics metrics, final Pattern pathPattern) { this.metrics = metrics; this.pathPattern = pathPattern; - this.deleteBlobRequests = createSensor("blob-delete"); - this.uploadBlobRequests = createSensor("blob-upload"); - this.uploadBlockRequests = createSensor("block-upload"); - this.uploadBlockListRequests = createSensor("block-list-upload"); - this.getBlobRequests = createSensor("blob-get"); + this.deleteBlobRequests = createSensor( + BLOB_DELETE, + BLOB_DELETE_RATE_METRIC_NAME, + BLOB_DELETE_TOTAL_METRIC_NAME + ); + this.uploadBlobRequests = createSensor( + BLOB_UPLOAD, + BLOB_UPLOAD_RATE_METRIC_NAME, + BLOB_UPLOAD_TOTAL_METRIC_NAME + ); + this.uploadBlockRequests = createSensor( + BLOCK_UPLOAD, + BLOCK_UPLOAD_RATE_METRIC_NAME, + BLOCK_UPLOAD_TOTAL_METRIC_NAME + ); + this.uploadBlockListRequests = createSensor( + BLOCK_LIST_UPLOAD, + BLOCK_LIST_UPLOAD_RATE_METRIC_NAME, + BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME + ); + this.getBlobRequests = createSensor( + BLOB_GET, + BLOB_GET_RATE_METRIC_NAME, + BLOB_GET_TOTAL_METRIC_NAME + ); } - private Sensor createSensor(final String name) { + private Sensor createSensor( + final String name, + final MetricNameTemplate rateMetricName, + final MetricNameTemplate totalMetricName + ) { return new SensorProvider(metrics, name) - .with(new MetricNameTemplate(name + "-rate", METRIC_GROUP, ""), new Rate()) - .with(new MetricNameTemplate(name + "-total", METRIC_GROUP, ""), new CumulativeCount()) + .with(rateMetricName, new Rate()) + .with(totalMetricName, new CumulativeCount()) .get(); } diff --git a/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java new file mode 100644 index 000000000..fdba497c5 --- /dev/null +++ b/storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java @@ -0,0 +1,75 @@ +/* + * Copyright 2024 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.tieredstorage.storage.azure; + +import java.util.List; + +import org.apache.kafka.common.MetricNameTemplate; + +public class MetricRegistry { + static final String METRIC_GROUP = "azure-blob-storage-client-metrics"; + static final String BLOB_DELETE = "blob-delete"; + static final String BLOB_DELETE_RATE = BLOB_DELETE + "-rate"; + static final String BLOB_DELETE_TOTAL = BLOB_DELETE + "-total"; + static final String BLOB_UPLOAD = "blob-upload"; + static final String BLOB_UPLOAD_RATE = BLOB_UPLOAD + "-rate"; + static final String BLOB_UPLOAD_TOTAL = BLOB_UPLOAD + "-total"; + static final String BLOCK_UPLOAD = "block-upload"; + static final String BLOCK_UPLOAD_RATE = BLOCK_UPLOAD + "-rate"; + static final String BLOCK_UPLOAD_TOTAL = BLOCK_UPLOAD + "-total"; + static final String BLOCK_LIST_UPLOAD = "block-list-upload"; + static final String BLOCK_LIST_UPLOAD_RATE = BLOCK_LIST_UPLOAD + "-rate"; + static final String BLOCK_LIST_UPLOAD_TOTAL = BLOCK_LIST_UPLOAD + "-total"; + static final String BLOB_GET = "blob-get"; + static final String BLOB_GET_RATE = BLOB_GET + "-rate"; + static final String BLOB_GET_TOTAL = BLOB_GET + "-total"; + + static final MetricNameTemplate BLOB_DELETE_RATE_METRIC_NAME = + new MetricNameTemplate(BLOB_DELETE_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate BLOB_DELETE_TOTAL_METRIC_NAME = + new MetricNameTemplate(BLOB_DELETE_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate BLOB_UPLOAD_RATE_METRIC_NAME = + new MetricNameTemplate(BLOB_UPLOAD_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate BLOB_UPLOAD_TOTAL_METRIC_NAME = + new MetricNameTemplate(BLOB_UPLOAD_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate BLOCK_UPLOAD_RATE_METRIC_NAME = + new MetricNameTemplate(BLOCK_UPLOAD_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate BLOCK_UPLOAD_TOTAL_METRIC_NAME = + new MetricNameTemplate(BLOCK_UPLOAD_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate BLOCK_LIST_UPLOAD_RATE_METRIC_NAME = + new MetricNameTemplate(BLOCK_LIST_UPLOAD_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME = + new MetricNameTemplate(BLOCK_LIST_UPLOAD_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate BLOB_GET_RATE_METRIC_NAME = new MetricNameTemplate(BLOB_GET_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate BLOB_GET_TOTAL_METRIC_NAME = + new MetricNameTemplate(BLOB_GET_TOTAL, METRIC_GROUP, ""); + + public List all() { + return List.of( + BLOB_DELETE_RATE_METRIC_NAME, + BLOB_DELETE_TOTAL_METRIC_NAME, + BLOB_UPLOAD_RATE_METRIC_NAME, + BLOB_UPLOAD_TOTAL_METRIC_NAME, + BLOCK_UPLOAD_RATE_METRIC_NAME, + BLOCK_UPLOAD_TOTAL_METRIC_NAME, + BLOCK_LIST_UPLOAD_RATE_METRIC_NAME, + BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME, + BLOB_GET_RATE_METRIC_NAME, + BLOB_GET_TOTAL_METRIC_NAME + ); + } +} diff --git a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java index 12a8a6e52..b57825bbd 100644 --- a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java +++ b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.regex.Pattern; +import org.apache.kafka.common.MetricNameTemplate; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.KafkaMetricsContext; import org.apache.kafka.common.metrics.MetricConfig; @@ -37,7 +38,23 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.http.HttpTransportOptions; -class MetricCollector { +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME; + +public class MetricCollector { private final org.apache.kafka.common.metrics.Metrics metrics; /** @@ -64,15 +81,13 @@ class MetricCollector { static final Pattern OBJECT_UPLOAD_PATH_PATTERN = Pattern.compile("^/upload/storage/v1/b/([^/]+)/o/?$"); - private static final String METRIC_GROUP = "gcs-client-metrics"; - private final Sensor getObjectMetadataRequests; private final Sensor deleteObjectRequests; private final Sensor resumableUploadInitiateRequests; private final Sensor resumableChunkUploadRequests; private final Sensor getObjectRequests; - MetricCollector() { + public MetricCollector() { final JmxReporter reporter = new JmxReporter(); metrics = new org.apache.kafka.common.metrics.Metrics( @@ -80,17 +95,41 @@ class MetricCollector { new KafkaMetricsContext("aiven.kafka.server.tieredstorage.gcs") ); - getObjectMetadataRequests = createSensor("object-metadata-get"); - getObjectRequests = createSensor("object-get"); - deleteObjectRequests = createSensor("object-delete"); - resumableUploadInitiateRequests = createSensor("resumable-upload-initiate"); - resumableChunkUploadRequests = createSensor("resumable-chunk-upload"); + getObjectMetadataRequests = createSensor( + OBJECT_METADATA_GET, + OBJECT_METADATA_GET_RATE_METRIC_NAME, + OBJECT_METADATA_GET_TOTAL_METRIC_NAME + ); + getObjectRequests = createSensor( + OBJECT_GET, + OBJECT_GET_RATE_METRIC_NAME, + OBJECT_GET_TOTAL_METRIC_NAME + ); + deleteObjectRequests = createSensor( + OBJECT_DELETE, + OBJECT_DELETE_RATE_METRIC_NAME, + OBJECT_DELETE_TOTAL_METRIC_NAME + ); + resumableUploadInitiateRequests = createSensor( + RESUMABLE_UPLOAD_INITIATE, + RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME, + RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME + ); + resumableChunkUploadRequests = createSensor( + RESUMABLE_CHUNK_UPLOAD, + RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME, + RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME + ); } - private Sensor createSensor(final String name) { + private Sensor createSensor( + final String name, + final MetricNameTemplate rateMetricName, + final MetricNameTemplate totalMetricName + ) { final Sensor sensor = metrics.sensor(name); - sensor.add(metrics.metricName(name + "-rate", METRIC_GROUP), new Rate()); - sensor.add(metrics.metricName(name + "-total", METRIC_GROUP), new CumulativeCount()); + sensor.add(metrics.metricInstance(rateMetricName), new Rate()); + sensor.add(metrics.metricInstance(totalMetricName), new CumulativeCount()); return sensor; } diff --git a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java new file mode 100644 index 000000000..1028a1557 --- /dev/null +++ b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java @@ -0,0 +1,76 @@ +/* + * Copyright 2024 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.tieredstorage.storage.gcs; + +import java.util.List; + +import org.apache.kafka.common.MetricNameTemplate; + +public class MetricRegistry { + static final String METRIC_GROUP = "gcs-client-metrics"; + static final String OBJECT_METADATA_GET = "object-metadata-get"; + static final String OBJECT_METADATA_GET_RATE = OBJECT_METADATA_GET + "-rate"; + static final String OBJECT_METADATA_GET_TOTAL = OBJECT_METADATA_GET + "-total"; + static final String OBJECT_GET = "object-get"; + static final String OBJECT_GET_RATE = OBJECT_GET + "-rate"; + static final String OBJECT_GET_TOTAL = OBJECT_GET + "-total"; + static final String OBJECT_DELETE = "object-delete"; + static final String OBJECT_DELETE_RATE = OBJECT_DELETE + "-rate"; + static final String OBJECT_DELETE_TOTAL = OBJECT_DELETE + "-total"; + static final String RESUMABLE_UPLOAD_INITIATE = "resumable-upload-initiate"; + static final String RESUMABLE_UPLOAD_INITIATE_RATE = RESUMABLE_UPLOAD_INITIATE + "-rate"; + static final String RESUMABLE_UPLOAD_INITIATE_TOTAL = RESUMABLE_UPLOAD_INITIATE + "-total"; + static final String RESUMABLE_CHUNK_UPLOAD = "resumable-chunk-upload"; + static final String RESUMABLE_CHUNK_UPLOAD_RATE = RESUMABLE_CHUNK_UPLOAD + "-rate"; + static final String RESUMABLE_CHUNK_UPLOAD_TOTAL = RESUMABLE_CHUNK_UPLOAD + "-total"; + + static final MetricNameTemplate OBJECT_METADATA_GET_RATE_METRIC_NAME = + new MetricNameTemplate(OBJECT_METADATA_GET_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate OBJECT_METADATA_GET_TOTAL_METRIC_NAME = + new MetricNameTemplate(OBJECT_METADATA_GET_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate OBJECT_GET_RATE_METRIC_NAME = + new MetricNameTemplate(OBJECT_GET_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate OBJECT_GET_TOTAL_METRIC_NAME = + new MetricNameTemplate(OBJECT_GET_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate OBJECT_DELETE_RATE_METRIC_NAME = + new MetricNameTemplate(OBJECT_DELETE_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate OBJECT_DELETE_TOTAL_METRIC_NAME = + new MetricNameTemplate(OBJECT_DELETE_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME = + new MetricNameTemplate(RESUMABLE_UPLOAD_INITIATE_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME = + new MetricNameTemplate(RESUMABLE_UPLOAD_INITIATE_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME = + new MetricNameTemplate(RESUMABLE_CHUNK_UPLOAD_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME = + new MetricNameTemplate(RESUMABLE_CHUNK_UPLOAD_TOTAL, METRIC_GROUP, ""); + + public List all() { + return List.of( + OBJECT_METADATA_GET_RATE_METRIC_NAME, + OBJECT_METADATA_GET_TOTAL_METRIC_NAME, + OBJECT_GET_RATE_METRIC_NAME, + OBJECT_GET_TOTAL_METRIC_NAME, + OBJECT_DELETE_RATE_METRIC_NAME, + OBJECT_DELETE_TOTAL_METRIC_NAME, + RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME, + RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME, + RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME, + RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME + ); + } +} diff --git a/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java index b90d73f21..4a45b4714 100644 --- a/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java +++ b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.stream.Collectors; +import org.apache.kafka.common.MetricNameTemplate; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.KafkaMetricsContext; import org.apache.kafka.common.metrics.MetricConfig; @@ -39,64 +40,239 @@ import software.amazon.awssdk.metrics.MetricCollection; import software.amazon.awssdk.metrics.MetricPublisher; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CONFIGURED_TIMEOUT_ERRORS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CONFIGURED_TIMEOUT_ERRORS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CONFIGURED_TIMEOUT_ERRORS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_TIME_MAX_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_TIME_MAX_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_TIME_MAX_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_TIME_MAX_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.SERVER_ERRORS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.SERVER_ERRORS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.SERVER_ERRORS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.THROTTLING_ERRORS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.THROTTLING_ERRORS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.THROTTLING_ERRORS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_REQUESTS; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_REQUESTS_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_REQUESTS_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_TIME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_TIME_AVG_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_TIME_MAX_METRIC_NAME; import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.CONFIGURED_TIMEOUT; import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.IO; import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.OTHER; import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.SERVER_ERROR; import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.THROTTLING; -class MetricCollector implements MetricPublisher { +public class MetricCollector implements MetricPublisher { private static final Logger log = LoggerFactory.getLogger(MetricCollector.class); private final org.apache.kafka.common.metrics.Metrics metrics; - private static final String METRIC_GROUP = "s3-client-metrics"; private final Map requestMetrics = new HashMap<>(); private final Map latencyMetrics = new HashMap<>(); private final Map errorMetrics = new HashMap<>(); - MetricCollector() { + public MetricCollector() { final MetricsReporter reporter = new JmxReporter(); metrics = new org.apache.kafka.common.metrics.Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, new KafkaMetricsContext("aiven.kafka.server.tieredstorage.s3") ); - requestMetrics.put("GetObject", createRequestsSensor("get-object-requests")); - latencyMetrics.put("GetObject", createLatencySensor("get-object-time")); - requestMetrics.put("UploadPart", createRequestsSensor("upload-part-requests")); - latencyMetrics.put("UploadPart", createLatencySensor("upload-part-time")); - requestMetrics.put("CreateMultipartUpload", createRequestsSensor("create-multipart-upload-requests")); - latencyMetrics.put("CreateMultipartUpload", createLatencySensor("create-multipart-upload-time")); - requestMetrics.put("CompleteMultipartUpload", createRequestsSensor("complete-multipart-upload-requests")); - latencyMetrics.put("CompleteMultipartUpload", createLatencySensor("complete-multipart-upload-time")); - requestMetrics.put("PutObject", createRequestsSensor("put-object-requests")); - latencyMetrics.put("PutObject", createLatencySensor("put-object-time")); - requestMetrics.put("DeleteObject", createRequestsSensor("delete-object-requests")); - latencyMetrics.put("DeleteObject", createLatencySensor("delete-object-time")); - requestMetrics.put("DeleteObjects", createRequestsSensor("delete-objects-requests")); - latencyMetrics.put("DeleteObjects", createLatencySensor("delete-objects-time")); - requestMetrics.put("AbortMultipartUpload", createRequestsSensor("abort-multipart-upload-requests")); - latencyMetrics.put("AbortMultipartUpload", createLatencySensor("abort-multipart-upload-time")); - - errorMetrics.put(THROTTLING.toString(), createRequestsSensor("throttling-errors")); - errorMetrics.put(SERVER_ERROR.toString(), createRequestsSensor("server-errors")); - errorMetrics.put(CONFIGURED_TIMEOUT.toString(), createRequestsSensor("configured-timeout-errors")); - errorMetrics.put(IO.toString(), createRequestsSensor("io-errors")); - errorMetrics.put(OTHER.toString(), createRequestsSensor("other-errors")); + final Sensor getObjectRequestsSensor = createRequestsSensor( + GET_OBJECT_REQUESTS, + GET_OBJECT_REQUESTS_RATE_METRIC_NAME, + GET_OBJECT_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("GetObject", getObjectRequestsSensor); + final Sensor getObjectTimeSensor = createLatencySensor( + GET_OBJECT_TIME, + GET_OBJECT_TIME_AVG_METRIC_NAME, + GET_OBJECT_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("GetObject", getObjectTimeSensor); + final Sensor uploadPartRequestsSensor = createRequestsSensor( + UPLOAD_PART_REQUESTS, + UPLOAD_PART_REQUESTS_RATE_METRIC_NAME, + UPLOAD_PART_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("UploadPart", uploadPartRequestsSensor); + final Sensor uploadPartTimeSensor = createLatencySensor( + UPLOAD_PART_TIME, + UPLOAD_PART_TIME_AVG_METRIC_NAME, + UPLOAD_PART_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("UploadPart", uploadPartTimeSensor); + final Sensor createMpuRequestsSensor = createRequestsSensor( + CREATE_MULTIPART_UPLOAD_REQUESTS, + CREATE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME, + CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("CreateMultipartUpload", createMpuRequestsSensor); + final Sensor createMpuTimeSensor = createLatencySensor( + CREATE_MULTIPART_UPLOAD_TIME, + CREATE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME, + CREATE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("CreateMultipartUpload", createMpuTimeSensor); + final Sensor completeMpuRequestsSensor = createRequestsSensor( + COMPLETE_MULTIPART_UPLOAD_REQUESTS, + COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME, + COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("CompleteMultipartUpload", completeMpuRequestsSensor); + final Sensor completeMpuTimeSensor = createLatencySensor( + COMPLETE_MULTIPART_UPLOAD_TIME, + COMPLETE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME, + COMPLETE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("CompleteMultipartUpload", completeMpuTimeSensor); + final Sensor putObjectRequestsSensor = createRequestsSensor( + PUT_OBJECT_REQUESTS, + PUT_OBJECT_REQUESTS_RATE_METRIC_NAME, + PUT_OBJECT_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("PutObject", putObjectRequestsSensor); + final Sensor putObjectTimeSensor = createLatencySensor( + PUT_OBJECT_TIME, + PUT_OBJECT_TIME_AVG_METRIC_NAME, + PUT_OBJECT_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("PutObject", putObjectTimeSensor); + final Sensor deleteObjectRequestsSensor = createRequestsSensor( + DELETE_OBJECT_REQUESTS, + DELETE_OBJECT_REQUESTS_RATE_METRIC_NAME, + DELETE_OBJECT_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("DeleteObject", deleteObjectRequestsSensor); + final Sensor deleteObjectTimeSensor = createLatencySensor( + DELETE_OBJECT_TIME, + DELETE_OBJECT_TIME_AVG_METRIC_NAME, + DELETE_OBJECT_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("DeleteObject", deleteObjectTimeSensor); + final Sensor deleteObjectsRequestsSensor = createRequestsSensor( + DELETE_OBJECTS_REQUESTS, + DELETE_OBJECTS_REQUESTS_RATE_METRIC_NAME, + DELETE_OBJECTS_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("DeleteObjects", deleteObjectsRequestsSensor); + final Sensor deleteObjectsTimeSensor = createLatencySensor( + DELETE_OBJECTS_TIME, + DELETE_OBJECTS_TIME_AVG_METRIC_NAME, + DELETE_OBJECTS_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("DeleteObjects", deleteObjectsTimeSensor); + final Sensor abortMpuRequestsSensor = createRequestsSensor( + ABORT_MULTIPART_UPLOAD_REQUESTS, + ABORT_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME, + ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME + ); + requestMetrics.put("AbortMultipartUpload", abortMpuRequestsSensor); + final Sensor abortMpuTimeSensor = createLatencySensor( + ABORT_MULTIPART_UPLOAD_TIME, + ABORT_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME, + ABORT_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME + ); + latencyMetrics.put("AbortMultipartUpload", abortMpuTimeSensor); + + final Sensor throttlingErrorsSensor = createRequestsSensor( + THROTTLING_ERRORS, + THROTTLING_ERRORS_RATE_METRIC_NAME, + THROTTLING_ERRORS_TOTAL_METRIC_NAME + ); + errorMetrics.put(THROTTLING.toString(), throttlingErrorsSensor); + final Sensor serverErrorsSensor = createRequestsSensor( + SERVER_ERRORS, + SERVER_ERRORS_RATE_METRIC_NAME, + SERVER_ERRORS_TOTAL_METRIC_NAME + ); + errorMetrics.put(SERVER_ERROR.toString(), serverErrorsSensor); + final Sensor configuredTimeoutErrorsSensor = createRequestsSensor( + CONFIGURED_TIMEOUT_ERRORS, + CONFIGURED_TIMEOUT_ERRORS_RATE_METRIC_NAME, + CONFIGURED_TIMEOUT_ERRORS_TOTAL_METRIC_NAME + ); + errorMetrics.put(CONFIGURED_TIMEOUT.toString(), configuredTimeoutErrorsSensor); + final Sensor ioErrorsSensor = createRequestsSensor( + IO_ERRORS, + IO_ERRORS_RATE_METRIC_NAME, + IO_ERRORS_TOTAL_METRIC_NAME + ); + errorMetrics.put(IO.toString(), ioErrorsSensor); + final Sensor otherErrorsSensor = createRequestsSensor( + OTHER_ERRORS, + OTHER_ERRORS_RATE_METRIC_NAME, + OTHER_ERRORS_TOTAL_METRIC_NAME + ); + errorMetrics.put(OTHER.toString(), otherErrorsSensor); } - private Sensor createRequestsSensor(final String name) { + private Sensor createRequestsSensor( + final String name, + final MetricNameTemplate rateMetricName, + final MetricNameTemplate totalMetricName + ) { final Sensor sensor = metrics.sensor(name); - sensor.add(metrics.metricName(name + "-rate", METRIC_GROUP), new Rate()); - sensor.add(metrics.metricName(name + "-total", METRIC_GROUP), new CumulativeCount()); + sensor.add(metrics.metricInstance(rateMetricName), new Rate()); + sensor.add(metrics.metricInstance(totalMetricName), new CumulativeCount()); return sensor; } - private Sensor createLatencySensor(final String name) { + private Sensor createLatencySensor( + final String name, + final MetricNameTemplate avgMetricName, + final MetricNameTemplate maxMetricName + ) { final Sensor sensor = metrics.sensor(name); - sensor.add(metrics.metricName(name + "-max", METRIC_GROUP), new Max()); - sensor.add(metrics.metricName(name + "-avg", METRIC_GROUP), new Avg()); + sensor.add(metrics.metricInstance(maxMetricName), new Max()); + sensor.add(metrics.metricInstance(avgMetricName), new Avg()); return sensor; } diff --git a/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java new file mode 100644 index 000000000..98b8e843c --- /dev/null +++ b/storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricRegistry.java @@ -0,0 +1,222 @@ +/* + * Copyright 2024 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.tieredstorage.storage.s3; + +import java.util.List; + +import org.apache.kafka.common.MetricNameTemplate; + +public class MetricRegistry { + static final String METRIC_GROUP = "s3-client-metrics"; + static final String GET_OBJECT_REQUESTS = "get-object-requests"; + static final String GET_OBJECT_REQUESTS_RATE = GET_OBJECT_REQUESTS + "-rate"; + static final String GET_OBJECT_REQUESTS_TOTAL = GET_OBJECT_REQUESTS + "-total"; + static final String GET_OBJECT_TIME = "get-object-time"; + static final String GET_OBJECT_TIME_AVG = GET_OBJECT_TIME + "-avg"; + static final String GET_OBJECT_TIME_MAX = GET_OBJECT_TIME + "-max"; + static final String UPLOAD_PART_REQUESTS = "upload-part-requests"; + static final String UPLOAD_PART_REQUESTS_RATE = UPLOAD_PART_REQUESTS + "-rate"; + static final String UPLOAD_PART_REQUESTS_TOTAL = UPLOAD_PART_REQUESTS + "-total"; + static final String UPLOAD_PART_TIME = "upload-part-time"; + static final String UPLOAD_PART_TIME_AVG = UPLOAD_PART_TIME + "-avg"; + static final String UPLOAD_PART_TIME_MAX = UPLOAD_PART_TIME + "-max"; + static final String PUT_OBJECT_REQUESTS = "put-object-requests"; + static final String PUT_OBJECT_REQUESTS_RATE = PUT_OBJECT_REQUESTS + "-rate"; + static final String PUT_OBJECT_REQUESTS_TOTAL = PUT_OBJECT_REQUESTS + "-total"; + static final String PUT_OBJECT_TIME = "put-object-time"; + static final String PUT_OBJECT_TIME_AVG = PUT_OBJECT_TIME + "-avg"; + static final String PUT_OBJECT_TIME_MAX = PUT_OBJECT_TIME + "-max"; + static final String DELETE_OBJECT_REQUESTS = "delete-object-requests"; + static final String DELETE_OBJECT_REQUESTS_RATE = DELETE_OBJECT_REQUESTS + "-rate"; + static final String DELETE_OBJECT_REQUESTS_TOTAL = DELETE_OBJECT_REQUESTS + "-total"; + static final String DELETE_OBJECT_TIME = "delete-object-time"; + static final String DELETE_OBJECT_TIME_AVG = DELETE_OBJECT_TIME + "-avg"; + static final String DELETE_OBJECT_TIME_MAX = DELETE_OBJECT_TIME + "-max"; + static final String DELETE_OBJECTS_REQUESTS = "delete-objects-requests"; + static final String DELETE_OBJECTS_REQUESTS_RATE = DELETE_OBJECTS_REQUESTS + "-rate"; + static final String DELETE_OBJECTS_REQUESTS_TOTAL = DELETE_OBJECTS_REQUESTS + "-total"; + static final String DELETE_OBJECTS_TIME = "delete-objects-time"; + static final String DELETE_OBJECTS_TIME_AVG = DELETE_OBJECTS_TIME + "-avg"; + static final String DELETE_OBJECTS_TIME_MAX = DELETE_OBJECTS_TIME + "-max"; + static final String CREATE_MULTIPART_UPLOAD_REQUESTS = "create-multipart-upload-requests"; + static final String CREATE_MULTIPART_UPLOAD_REQUESTS_RATE = CREATE_MULTIPART_UPLOAD_REQUESTS + "-rate"; + static final String CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL = CREATE_MULTIPART_UPLOAD_REQUESTS + "-total"; + static final String CREATE_MULTIPART_UPLOAD_TIME = "create-multipart-upload-time"; + static final String CREATE_MULTIPART_UPLOAD_TIME_AVG = CREATE_MULTIPART_UPLOAD_TIME + "-avg"; + static final String CREATE_MULTIPART_UPLOAD_TIME_MAX = CREATE_MULTIPART_UPLOAD_TIME + "-max"; + static final String COMPLETE_MULTIPART_UPLOAD_REQUESTS = "complete-multipart-upload-requests"; + static final String COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE = COMPLETE_MULTIPART_UPLOAD_REQUESTS + "-rate"; + static final String COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL = COMPLETE_MULTIPART_UPLOAD_REQUESTS + "-total"; + static final String COMPLETE_MULTIPART_UPLOAD_TIME = "complete-multipart-upload-time"; + static final String COMPLETE_MULTIPART_UPLOAD_TIME_AVG = COMPLETE_MULTIPART_UPLOAD_TIME + "-avg"; + static final String COMPLETE_MULTIPART_UPLOAD_TIME_MAX = COMPLETE_MULTIPART_UPLOAD_TIME + "-max"; + static final String ABORT_MULTIPART_UPLOAD_REQUESTS = "abort-multipart-upload-requests"; + static final String ABORT_MULTIPART_UPLOAD_REQUESTS_RATE = ABORT_MULTIPART_UPLOAD_REQUESTS + "-rate"; + static final String ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL = ABORT_MULTIPART_UPLOAD_REQUESTS + "-total"; + static final String ABORT_MULTIPART_UPLOAD_TIME = "abort-multipart-upload-time"; + static final String ABORT_MULTIPART_UPLOAD_TIME_AVG = ABORT_MULTIPART_UPLOAD_TIME + "-avg"; + static final String ABORT_MULTIPART_UPLOAD_TIME_MAX = ABORT_MULTIPART_UPLOAD_TIME + "-max"; + + static final MetricNameTemplate GET_OBJECT_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(GET_OBJECT_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate GET_OBJECT_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(GET_OBJECT_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate GET_OBJECT_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(GET_OBJECT_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate GET_OBJECT_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(GET_OBJECT_TIME_AVG, METRIC_GROUP, ""); + static final MetricNameTemplate UPLOAD_PART_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(UPLOAD_PART_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate UPLOAD_PART_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(UPLOAD_PART_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate UPLOAD_PART_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(UPLOAD_PART_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate UPLOAD_PART_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(UPLOAD_PART_TIME_AVG, METRIC_GROUP, ""); + static final MetricNameTemplate PUT_OBJECT_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(PUT_OBJECT_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate PUT_OBJECT_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(PUT_OBJECT_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate PUT_OBJECT_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(PUT_OBJECT_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate PUT_OBJECT_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(PUT_OBJECT_TIME_AVG, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECT_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECT_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECT_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECT_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECT_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECT_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECT_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECT_TIME_AVG, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECTS_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECTS_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECTS_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECTS_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECTS_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECTS_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate DELETE_OBJECTS_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(DELETE_OBJECTS_TIME_AVG, METRIC_GROUP, ""); + static final MetricNameTemplate CREATE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(CREATE_MULTIPART_UPLOAD_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate CREATE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(CREATE_MULTIPART_UPLOAD_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate CREATE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(CREATE_MULTIPART_UPLOAD_TIME_AVG, METRIC_GROUP, ""); + static final MetricNameTemplate COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate COMPLETE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(COMPLETE_MULTIPART_UPLOAD_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate COMPLETE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(COMPLETE_MULTIPART_UPLOAD_TIME_AVG, METRIC_GROUP, ""); + static final MetricNameTemplate ABORT_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME = + new MetricNameTemplate(ABORT_MULTIPART_UPLOAD_REQUESTS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME = + new MetricNameTemplate(ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate ABORT_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME = + new MetricNameTemplate(ABORT_MULTIPART_UPLOAD_TIME_MAX, METRIC_GROUP, ""); + static final MetricNameTemplate ABORT_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME = + new MetricNameTemplate(ABORT_MULTIPART_UPLOAD_TIME_AVG, METRIC_GROUP, ""); + + static final String THROTTLING_ERRORS = "throttling-errors"; + static final String THROTTLING_ERRORS_RATE = THROTTLING_ERRORS + "-rate"; + static final String THROTTLING_ERRORS_TOTAL = THROTTLING_ERRORS + "-total"; + static final String SERVER_ERRORS = "server-errors"; + static final String SERVER_ERRORS_RATE = SERVER_ERRORS + "-rate"; + static final String SERVER_ERRORS_TOTAL = SERVER_ERRORS + "-total"; + static final String CONFIGURED_TIMEOUT_ERRORS = "configured-timeout-errors"; + static final String CONFIGURED_TIMEOUT_ERRORS_RATE = CONFIGURED_TIMEOUT_ERRORS + "-rate"; + static final String CONFIGURED_TIMEOUT_ERRORS_TOTAL = CONFIGURED_TIMEOUT_ERRORS + "-total"; + static final String IO_ERRORS = "io-errors"; + static final String IO_ERRORS_RATE = IO_ERRORS + "-rate"; + static final String IO_ERRORS_TOTAL = IO_ERRORS + "-total"; + static final String OTHER_ERRORS = "other-errors"; + static final String OTHER_ERRORS_RATE = OTHER_ERRORS + "-rate"; + static final String OTHER_ERRORS_TOTAL = OTHER_ERRORS + "-total"; + + static final MetricNameTemplate THROTTLING_ERRORS_RATE_METRIC_NAME = + new MetricNameTemplate(THROTTLING_ERRORS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate THROTTLING_ERRORS_TOTAL_METRIC_NAME = + new MetricNameTemplate(THROTTLING_ERRORS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate SERVER_ERRORS_RATE_METRIC_NAME = + new MetricNameTemplate(SERVER_ERRORS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate SERVER_ERRORS_TOTAL_METRIC_NAME = + new MetricNameTemplate(SERVER_ERRORS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate CONFIGURED_TIMEOUT_ERRORS_RATE_METRIC_NAME = + new MetricNameTemplate(CONFIGURED_TIMEOUT_ERRORS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate CONFIGURED_TIMEOUT_ERRORS_TOTAL_METRIC_NAME = + new MetricNameTemplate(CONFIGURED_TIMEOUT_ERRORS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate IO_ERRORS_RATE_METRIC_NAME = + new MetricNameTemplate(IO_ERRORS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate IO_ERRORS_TOTAL_METRIC_NAME = + new MetricNameTemplate(IO_ERRORS_TOTAL, METRIC_GROUP, ""); + static final MetricNameTemplate OTHER_ERRORS_RATE_METRIC_NAME = + new MetricNameTemplate(OTHER_ERRORS_RATE, METRIC_GROUP, ""); + static final MetricNameTemplate OTHER_ERRORS_TOTAL_METRIC_NAME = + new MetricNameTemplate(OTHER_ERRORS_TOTAL, METRIC_GROUP, ""); + + public List all() { + return List.of( + GET_OBJECT_REQUESTS_RATE_METRIC_NAME, + GET_OBJECT_REQUESTS_TOTAL_METRIC_NAME, + GET_OBJECT_TIME_AVG_METRIC_NAME, + GET_OBJECT_TIME_MAX_METRIC_NAME, + UPLOAD_PART_REQUESTS_RATE_METRIC_NAME, + UPLOAD_PART_REQUESTS_TOTAL_METRIC_NAME, + UPLOAD_PART_TIME_AVG_METRIC_NAME, + UPLOAD_PART_TIME_MAX_METRIC_NAME, + PUT_OBJECT_REQUESTS_RATE_METRIC_NAME, + PUT_OBJECT_REQUESTS_TOTAL_METRIC_NAME, + PUT_OBJECT_TIME_AVG_METRIC_NAME, + PUT_OBJECT_TIME_MAX_METRIC_NAME, + DELETE_OBJECT_REQUESTS_RATE_METRIC_NAME, + DELETE_OBJECT_REQUESTS_TOTAL_METRIC_NAME, + DELETE_OBJECT_TIME_AVG_METRIC_NAME, + DELETE_OBJECT_TIME_MAX_METRIC_NAME, + DELETE_OBJECTS_REQUESTS_RATE_METRIC_NAME, + DELETE_OBJECTS_REQUESTS_TOTAL_METRIC_NAME, + DELETE_OBJECTS_TIME_AVG_METRIC_NAME, + DELETE_OBJECTS_TIME_MAX_METRIC_NAME, + CREATE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME, + CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME, + CREATE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME, + CREATE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME, + COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME, + COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME, + COMPLETE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME, + COMPLETE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME, + ABORT_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME, + ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME, + ABORT_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME, + ABORT_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME, + THROTTLING_ERRORS_RATE_METRIC_NAME, + THROTTLING_ERRORS_TOTAL_METRIC_NAME, + SERVER_ERRORS_RATE_METRIC_NAME, + SERVER_ERRORS_TOTAL_METRIC_NAME, + CONFIGURED_TIMEOUT_ERRORS_RATE_METRIC_NAME, + CONFIGURED_TIMEOUT_ERRORS_TOTAL_METRIC_NAME, + IO_ERRORS_RATE_METRIC_NAME, + IO_ERRORS_TOTAL_METRIC_NAME, + OTHER_ERRORS_RATE_METRIC_NAME, + OTHER_ERRORS_TOTAL_METRIC_NAME + ); + } +}