From 37bc22a89fb45db9682e5cf415bfea78117b160a Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 22 Oct 2024 14:22:30 +0300 Subject: [PATCH] refactor: expose gcs metric names for doc generation --- .../storage/gcs/MetricCollector.java | 66 ++++++++-- .../storage/gcs/MetricRegistry.java | 115 ++++++++++++++++++ 2 files changed, 168 insertions(+), 13 deletions(-) create mode 100644 storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java diff --git a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java index 12a8a6e52..a85317bcb 100644 --- a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java +++ b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.regex.Pattern; +import org.apache.kafka.common.MetricNameTemplate; import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.KafkaMetricsContext; import org.apache.kafka.common.metrics.MetricConfig; @@ -37,7 +38,24 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.http.HttpTransportOptions; -class MetricCollector { +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME; +import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME; + +public class MetricCollector { private final org.apache.kafka.common.metrics.Metrics metrics; /** @@ -64,33 +82,55 @@ class MetricCollector { static final Pattern OBJECT_UPLOAD_PATH_PATTERN = Pattern.compile("^/upload/storage/v1/b/([^/]+)/o/?$"); - private static final String METRIC_GROUP = "gcs-client-metrics"; - private final Sensor getObjectMetadataRequests; private final Sensor deleteObjectRequests; private final Sensor resumableUploadInitiateRequests; private final Sensor resumableChunkUploadRequests; private final Sensor getObjectRequests; - MetricCollector() { + public MetricCollector() { final JmxReporter reporter = new JmxReporter(); metrics = new org.apache.kafka.common.metrics.Metrics( new MetricConfig(), List.of(reporter), Time.SYSTEM, - new KafkaMetricsContext("aiven.kafka.server.tieredstorage.gcs") + new KafkaMetricsContext(METRIC_CONTEXT) ); - getObjectMetadataRequests = createSensor("object-metadata-get"); - getObjectRequests = createSensor("object-get"); - deleteObjectRequests = createSensor("object-delete"); - resumableUploadInitiateRequests = createSensor("resumable-upload-initiate"); - resumableChunkUploadRequests = createSensor("resumable-chunk-upload"); + getObjectMetadataRequests = createSensor( + OBJECT_METADATA_GET, + OBJECT_METADATA_GET_RATE_METRIC_NAME, + OBJECT_METADATA_GET_TOTAL_METRIC_NAME + ); + getObjectRequests = createSensor( + OBJECT_GET, + OBJECT_GET_RATE_METRIC_NAME, + OBJECT_GET_TOTAL_METRIC_NAME + ); + deleteObjectRequests = createSensor( + OBJECT_DELETE, + OBJECT_DELETE_RATE_METRIC_NAME, + OBJECT_DELETE_TOTAL_METRIC_NAME + ); + resumableUploadInitiateRequests = createSensor( + RESUMABLE_UPLOAD_INITIATE, + RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME, + RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME + ); + resumableChunkUploadRequests = createSensor( + RESUMABLE_CHUNK_UPLOAD, + RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME, + RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME + ); } - private Sensor createSensor(final String name) { + private Sensor createSensor( + final String name, + final MetricNameTemplate rateMetricName, + final MetricNameTemplate totalMetricName + ) { final Sensor sensor = metrics.sensor(name); - sensor.add(metrics.metricName(name + "-rate", METRIC_GROUP), new Rate()); - sensor.add(metrics.metricName(name + "-total", METRIC_GROUP), new CumulativeCount()); + sensor.add(metrics.metricInstance(rateMetricName), new Rate()); + sensor.add(metrics.metricInstance(totalMetricName), new CumulativeCount()); return sensor; } diff --git a/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java new file mode 100644 index 000000000..2ceaf7bd3 --- /dev/null +++ b/storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java @@ -0,0 +1,115 @@ +/* + * Copyright 2024 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.tieredstorage.storage.gcs; + +import java.util.List; + +import org.apache.kafka.common.MetricNameTemplate; + +public class MetricRegistry { + public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.gcs"; + static final String METRIC_GROUP = "gcs-client-metrics"; + static final String OBJECT_METADATA_GET = "object-metadata-get"; + static final String OBJECT_METADATA_GET_RATE = OBJECT_METADATA_GET + "-rate"; + static final String OBJECT_METADATA_GET_TOTAL = OBJECT_METADATA_GET + "-total"; + static final String OBJECT_METADATA_GET_DOC = "get object metadata operations"; + static final String OBJECT_GET = "object-get"; + static final String OBJECT_GET_RATE = OBJECT_GET + "-rate"; + static final String OBJECT_GET_TOTAL = OBJECT_GET + "-total"; + static final String OBJECT_GET_DOC = "get object operations"; + static final String OBJECT_DELETE = "object-delete"; + static final String OBJECT_DELETE_RATE = OBJECT_DELETE + "-rate"; + static final String OBJECT_DELETE_TOTAL = OBJECT_DELETE + "-total"; + static final String OBJECT_DELETE_DOC = "delete object operations"; + static final String RESUMABLE_UPLOAD_INITIATE = "resumable-upload-initiate"; + static final String RESUMABLE_UPLOAD_INITIATE_RATE = RESUMABLE_UPLOAD_INITIATE + "-rate"; + static final String RESUMABLE_UPLOAD_INITIATE_TOTAL = RESUMABLE_UPLOAD_INITIATE + "-total"; + static final String RESUMABLE_UPLOAD_INITIATE_DOC = "initiate resumable upload operations"; + static final String RESUMABLE_CHUNK_UPLOAD = "resumable-chunk-upload"; + static final String RESUMABLE_CHUNK_UPLOAD_RATE = RESUMABLE_CHUNK_UPLOAD + "-rate"; + static final String RESUMABLE_CHUNK_UPLOAD_TOTAL = RESUMABLE_CHUNK_UPLOAD + "-total"; + static final String RESUMABLE_CHUNK_UPLOAD_DOC = "upload chunk operations as part of resumable upload"; + + public static final String RATE_DOC_PREFIX = "Rate of "; + public static final String TOTAL_DOC_PREFIX = "Total number of "; + + static final MetricNameTemplate OBJECT_METADATA_GET_RATE_METRIC_NAME = new MetricNameTemplate( + OBJECT_METADATA_GET_RATE, + METRIC_GROUP, + RATE_DOC_PREFIX + OBJECT_METADATA_GET_DOC + ); + static final MetricNameTemplate OBJECT_METADATA_GET_TOTAL_METRIC_NAME = new MetricNameTemplate( + OBJECT_METADATA_GET_TOTAL, + METRIC_GROUP, + TOTAL_DOC_PREFIX + OBJECT_METADATA_GET_DOC + ); + static final MetricNameTemplate OBJECT_GET_RATE_METRIC_NAME = new MetricNameTemplate( + OBJECT_GET_RATE, + METRIC_GROUP, + RATE_DOC_PREFIX + OBJECT_GET_DOC + ); + static final MetricNameTemplate OBJECT_GET_TOTAL_METRIC_NAME = new MetricNameTemplate( + OBJECT_GET_TOTAL, + METRIC_GROUP, + TOTAL_DOC_PREFIX + OBJECT_GET_DOC + ); + static final MetricNameTemplate OBJECT_DELETE_RATE_METRIC_NAME = new MetricNameTemplate( + OBJECT_DELETE_RATE, + METRIC_GROUP, + RATE_DOC_PREFIX + OBJECT_DELETE_DOC + ); + static final MetricNameTemplate OBJECT_DELETE_TOTAL_METRIC_NAME = new MetricNameTemplate( + OBJECT_DELETE_TOTAL, + METRIC_GROUP, + TOTAL_DOC_PREFIX + OBJECT_DELETE_DOC + ); + static final MetricNameTemplate RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME = new MetricNameTemplate( + RESUMABLE_UPLOAD_INITIATE_RATE, + METRIC_GROUP, + RATE_DOC_PREFIX + RESUMABLE_UPLOAD_INITIATE_DOC + ); + static final MetricNameTemplate RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME = new MetricNameTemplate( + RESUMABLE_UPLOAD_INITIATE_TOTAL, + METRIC_GROUP, + TOTAL_DOC_PREFIX + RESUMABLE_UPLOAD_INITIATE_DOC + ); + static final MetricNameTemplate RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME = new MetricNameTemplate( + RESUMABLE_CHUNK_UPLOAD_RATE, + METRIC_GROUP, + RATE_DOC_PREFIX + RESUMABLE_CHUNK_UPLOAD_DOC + ); + static final MetricNameTemplate RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME = new MetricNameTemplate( + RESUMABLE_CHUNK_UPLOAD_TOTAL, + METRIC_GROUP, + TOTAL_DOC_PREFIX + RESUMABLE_CHUNK_UPLOAD_DOC + ); + + public List all() { + return List.of( + OBJECT_METADATA_GET_RATE_METRIC_NAME, + OBJECT_METADATA_GET_TOTAL_METRIC_NAME, + OBJECT_GET_RATE_METRIC_NAME, + OBJECT_GET_TOTAL_METRIC_NAME, + OBJECT_DELETE_RATE_METRIC_NAME, + OBJECT_DELETE_TOTAL_METRIC_NAME, + RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME, + RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME, + RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME, + RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME + ); + } +}