Skip to content

Commit

Permalink
refactor: expose gcs metric names for doc generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeqo
jeqo committed Oct 22, 2024
1 parent 44fb77c commit 37bc22a
Show file tree
Hide file tree
Showing 2 changed files with 168 additions and 13 deletions.
66 changes: 53 additions & 13 deletions storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.List;
import java.util.regex.Pattern;

import org.apache.kafka.common.MetricNameTemplate;
import org.apache.kafka.common.metrics.JmxReporter;
import org.apache.kafka.common.metrics.KafkaMetricsContext;
import org.apache.kafka.common.metrics.MetricConfig;
Expand All @@ -37,7 +38,24 @@
import com.google.cloud.ServiceOptions;
import com.google.cloud.http.HttpTransportOptions;

class MetricCollector {
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_RATE_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_TOTAL_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_RATE_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_TOTAL_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_RATE_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_TOTAL_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME;
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME;

public class MetricCollector {
private final org.apache.kafka.common.metrics.Metrics metrics;

/**
Expand All @@ -64,33 +82,55 @@ class MetricCollector {
static final Pattern OBJECT_UPLOAD_PATH_PATTERN =
Pattern.compile("^/upload/storage/v1/b/([^/]+)/o/?$");

private static final String METRIC_GROUP = "gcs-client-metrics";

private final Sensor getObjectMetadataRequests;
private final Sensor deleteObjectRequests;
private final Sensor resumableUploadInitiateRequests;
private final Sensor resumableChunkUploadRequests;
private final Sensor getObjectRequests;

MetricCollector() {
public MetricCollector() {
final JmxReporter reporter = new JmxReporter();

metrics = new org.apache.kafka.common.metrics.Metrics(
new MetricConfig(), List.of(reporter), Time.SYSTEM,
new KafkaMetricsContext("aiven.kafka.server.tieredstorage.gcs")
new KafkaMetricsContext(METRIC_CONTEXT)
);

getObjectMetadataRequests = createSensor("object-metadata-get");
getObjectRequests = createSensor("object-get");
deleteObjectRequests = createSensor("object-delete");
resumableUploadInitiateRequests = createSensor("resumable-upload-initiate");
resumableChunkUploadRequests = createSensor("resumable-chunk-upload");
getObjectMetadataRequests = createSensor(
OBJECT_METADATA_GET,
OBJECT_METADATA_GET_RATE_METRIC_NAME,
OBJECT_METADATA_GET_TOTAL_METRIC_NAME
);
getObjectRequests = createSensor(
OBJECT_GET,
OBJECT_GET_RATE_METRIC_NAME,
OBJECT_GET_TOTAL_METRIC_NAME
);
deleteObjectRequests = createSensor(
OBJECT_DELETE,
OBJECT_DELETE_RATE_METRIC_NAME,
OBJECT_DELETE_TOTAL_METRIC_NAME
);
resumableUploadInitiateRequests = createSensor(
RESUMABLE_UPLOAD_INITIATE,
RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME,
RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME
);
resumableChunkUploadRequests = createSensor(
RESUMABLE_CHUNK_UPLOAD,
RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME,
RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME
);
}

private Sensor createSensor(final String name) {
private Sensor createSensor(
final String name,
final MetricNameTemplate rateMetricName,
final MetricNameTemplate totalMetricName
) {
final Sensor sensor = metrics.sensor(name);
sensor.add(metrics.metricName(name + "-rate", METRIC_GROUP), new Rate());
sensor.add(metrics.metricName(name + "-total", METRIC_GROUP), new CumulativeCount());
sensor.add(metrics.metricInstance(rateMetricName), new Rate());
sensor.add(metrics.metricInstance(totalMetricName), new CumulativeCount());
return sensor;
}

Expand Down
115 changes: 115 additions & 0 deletions storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricRegistry.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/*
* Copyright 2024 Aiven Oy
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.aiven.kafka.tieredstorage.storage.gcs;

import java.util.List;

import org.apache.kafka.common.MetricNameTemplate;

public class MetricRegistry {
public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.gcs";
static final String METRIC_GROUP = "gcs-client-metrics";
static final String OBJECT_METADATA_GET = "object-metadata-get";
static final String OBJECT_METADATA_GET_RATE = OBJECT_METADATA_GET + "-rate";
static final String OBJECT_METADATA_GET_TOTAL = OBJECT_METADATA_GET + "-total";
static final String OBJECT_METADATA_GET_DOC = "get object metadata operations";
static final String OBJECT_GET = "object-get";
static final String OBJECT_GET_RATE = OBJECT_GET + "-rate";
static final String OBJECT_GET_TOTAL = OBJECT_GET + "-total";
static final String OBJECT_GET_DOC = "get object operations";
static final String OBJECT_DELETE = "object-delete";
static final String OBJECT_DELETE_RATE = OBJECT_DELETE + "-rate";
static final String OBJECT_DELETE_TOTAL = OBJECT_DELETE + "-total";
static final String OBJECT_DELETE_DOC = "delete object operations";
static final String RESUMABLE_UPLOAD_INITIATE = "resumable-upload-initiate";
static final String RESUMABLE_UPLOAD_INITIATE_RATE = RESUMABLE_UPLOAD_INITIATE + "-rate";
static final String RESUMABLE_UPLOAD_INITIATE_TOTAL = RESUMABLE_UPLOAD_INITIATE + "-total";
static final String RESUMABLE_UPLOAD_INITIATE_DOC = "initiate resumable upload operations";
static final String RESUMABLE_CHUNK_UPLOAD = "resumable-chunk-upload";
static final String RESUMABLE_CHUNK_UPLOAD_RATE = RESUMABLE_CHUNK_UPLOAD + "-rate";
static final String RESUMABLE_CHUNK_UPLOAD_TOTAL = RESUMABLE_CHUNK_UPLOAD + "-total";
static final String RESUMABLE_CHUNK_UPLOAD_DOC = "upload chunk operations as part of resumable upload";

public static final String RATE_DOC_PREFIX = "Rate of ";
public static final String TOTAL_DOC_PREFIX = "Total number of ";

static final MetricNameTemplate OBJECT_METADATA_GET_RATE_METRIC_NAME = new MetricNameTemplate(
OBJECT_METADATA_GET_RATE,
METRIC_GROUP,
RATE_DOC_PREFIX + OBJECT_METADATA_GET_DOC
);
static final MetricNameTemplate OBJECT_METADATA_GET_TOTAL_METRIC_NAME = new MetricNameTemplate(
OBJECT_METADATA_GET_TOTAL,
METRIC_GROUP,
TOTAL_DOC_PREFIX + OBJECT_METADATA_GET_DOC
);
static final MetricNameTemplate OBJECT_GET_RATE_METRIC_NAME = new MetricNameTemplate(
OBJECT_GET_RATE,
METRIC_GROUP,
RATE_DOC_PREFIX + OBJECT_GET_DOC
);
static final MetricNameTemplate OBJECT_GET_TOTAL_METRIC_NAME = new MetricNameTemplate(
OBJECT_GET_TOTAL,
METRIC_GROUP,
TOTAL_DOC_PREFIX + OBJECT_GET_DOC
);
static final MetricNameTemplate OBJECT_DELETE_RATE_METRIC_NAME = new MetricNameTemplate(
OBJECT_DELETE_RATE,
METRIC_GROUP,
RATE_DOC_PREFIX + OBJECT_DELETE_DOC
);
static final MetricNameTemplate OBJECT_DELETE_TOTAL_METRIC_NAME = new MetricNameTemplate(
OBJECT_DELETE_TOTAL,
METRIC_GROUP,
TOTAL_DOC_PREFIX + OBJECT_DELETE_DOC
);
static final MetricNameTemplate RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME = new MetricNameTemplate(
RESUMABLE_UPLOAD_INITIATE_RATE,
METRIC_GROUP,
RATE_DOC_PREFIX + RESUMABLE_UPLOAD_INITIATE_DOC
);
static final MetricNameTemplate RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME = new MetricNameTemplate(
RESUMABLE_UPLOAD_INITIATE_TOTAL,
METRIC_GROUP,
TOTAL_DOC_PREFIX + RESUMABLE_UPLOAD_INITIATE_DOC
);
static final MetricNameTemplate RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME = new MetricNameTemplate(
RESUMABLE_CHUNK_UPLOAD_RATE,
METRIC_GROUP,
RATE_DOC_PREFIX + RESUMABLE_CHUNK_UPLOAD_DOC
);
static final MetricNameTemplate RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME = new MetricNameTemplate(
RESUMABLE_CHUNK_UPLOAD_TOTAL,
METRIC_GROUP,
TOTAL_DOC_PREFIX + RESUMABLE_CHUNK_UPLOAD_DOC
);

public List<MetricNameTemplate> all() {
return List.of(
OBJECT_METADATA_GET_RATE_METRIC_NAME,
OBJECT_METADATA_GET_TOTAL_METRIC_NAME,
OBJECT_GET_RATE_METRIC_NAME,
OBJECT_GET_TOTAL_METRIC_NAME,
OBJECT_DELETE_RATE_METRIC_NAME,
OBJECT_DELETE_TOTAL_METRIC_NAME,
RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME,
RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME,
RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME,
RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME
);
}
}

0 comments on commit 37bc22a

Please sign in to comment.