From 59c2422478856aaee92c3ecc7b9434f082e8483f Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Thu, 11 Apr 2024 17:38:28 -0700 Subject: [PATCH 01/29] poc, take 1 --- pom.xml | 1 + pulsar-tracing/pom.xml | 82 +++ .../oss/pulsar/tracing/BrokerTracing.java | 645 ++++++++++++++++++ .../META-INF/services/broker_interceptor.yml | 3 + pulsar-tracing/src/test/resources/log4j2.xml | 41 ++ 5 files changed, 772 insertions(+) create mode 100644 pulsar-tracing/pom.xml create mode 100644 pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java create mode 100644 pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml create mode 100644 pulsar-tracing/src/test/resources/log4j2.xml diff --git a/pom.xml b/pom.xml index 0e83aa9a..8f9c2802 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ activemq-filters pulsar-jms-filters + pulsar-tracing pulsar-jms-admin-api pulsar-jms resource-adapter diff --git a/pulsar-tracing/pom.xml b/pulsar-tracing/pom.xml new file mode 100644 index 00000000..e4a65161 --- /dev/null +++ b/pulsar-tracing/pom.xml @@ -0,0 +1,82 @@ + + + + + pulsar-jms-parent + com.datastax.oss + 4.1.1-alpha-SNAPSHOT + + 4.0.0 + pulsar-tracing + jar + DataStax Starlight for JMS - Broker Side Filters + + ${project.build.directory} + + + + org.projectlombok + lombok + provided + + + com.github.spotbugs + spotbugs-annotations + provided + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + ${pulsar.groupId} + pulsar-broker + provided + + + + + + org.apache.nifi + nifi-nar-maven-plugin + 1.3.2 + true + + pulsar-tracing-${project.version} + nar + + + + default-nar + package + + nar + + + + + + + diff --git a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java b/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java new file mode 100644 index 00000000..e18e2cec --- /dev/null +++ b/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java @@ -0,0 +1,645 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.pulsar.tracing; + +import io.netty.buffer.ByteBuf; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.commons.codec.binary.Hex; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Producer; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.common.api.proto.BaseCommand; +import org.apache.pulsar.common.api.proto.CommandAck; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.intercept.InterceptException; +import org.apache.pulsar.common.naming.TopicName; + +@Slf4j +public class BrokerTracing implements BrokerInterceptor { + + static final int MAX_DATA_LENGTH = 1024; + + public enum EventReasons { + ADMINISTRATIVE, + MESSAGE, + TRANSACTION, + SERVLET, + } + + public enum TraceLevel { + NONE, + MIMIMAL, + BASIC, + FULL + } + + // private final Set defaultEnabledEvents = new HashSet<>(); + private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC; + + public void initialize(PulsarService pulsarService) { + // defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE); + // defaultEnabledEvents.add(EventReasons.MESSAGE); + // defaultEnabledEvents.add(EventReasons.TRANSACTION); + } + + @Override + public void close() {} + + private Set getEnabledEvents(ServerCnx cnx) { + return getEnabledEvents(cnx.getBrokerService().getPulsar()); + } + + private Set getEnabledEvents(PulsarService pulsar) { + // todo: do this less frequently + + String events = pulsar.getConfiguration().getProperties().getProperty("tracing_event_list", ""); + + Set enabledEvents = new HashSet<>(); + + for (String event : events.split(",")) { + try { + enabledEvents.add(EventReasons.valueOf(event.trim())); + } catch (IllegalArgumentException e) { + log.warn("Invalid event: {}. Skipping", event); + } + } + + return enabledEvents; + } + + private static TraceLevel getTracingLevel(ServerCnx cnx) { + // todo: do this less frequently + String level = + cnx.getBrokerService() + .getPulsar() + .getConfiguration() + .getProperties() + .getProperty("tracing_level", defaultTraceLevel.toString()); + try { + return TraceLevel.valueOf(level); + } catch (IllegalArgumentException e) { + log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel); + return defaultTraceLevel; + } + } + + private static void populateConnectionDetails( + TraceLevel level, ServerCnx cnx, Map traceDetails) { + if (cnx == null) { + traceDetails.put("ServerCnx", "null"); + return; + } + + switch (level) { + case MIMIMAL: + traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString()); + break; + case BASIC: + populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails); + + traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion()); + traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); + break; + case FULL: + populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails); + populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); + + traceDetails.put("ServerCnx.authRole", cnx.getAuthRole()); + traceDetails.put("ServerCnx.authMethod", cnx.getAuthMethod()); + traceDetails.put( + "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); + break; + default: + log.debug("Unknown tracing level: {}", level); + break; + } + } + + private static void populateSubscriptionDetails( + TraceLevel level, Subscription sub, Map traceDetails) { + if (sub == null) { + traceDetails.put("Subscription", "null"); + return; + } + + switch (level) { + case MIMIMAL: + traceDetails.put("Subscription.name", sub.getName()); + traceDetails.put( + "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); + traceDetails.put("Subscription.type", sub.getType().name()); + break; + case BASIC: + populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails); + + if (sub.getConsumers() != null) { + traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size()); + traceDetails.put( + "Subscription.namesOfConsumers", + sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList())); + } + + break; + case FULL: + populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails); + populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails); + + traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties()); + break; + default: + log.debug("Unknown tracing level: {}", level); + break; + } + } + + private static void populateConsumerDetails( + TraceLevel level, Consumer consumer, Map traceDetails) { + if (consumer == null) { + traceDetails.put("Consumer", "null"); + return; + } + + switch (level) { + case MIMIMAL: + traceDetails.put("Consumer.name", consumer.consumerName()); + traceDetails.put("Consumer.consumerId", consumer.consumerId()); + if (consumer.getSubscription() != null) { + traceDetails.put("Consumer.subscriptionName", consumer.getSubscription().getName()); + traceDetails.put( + "Consumer.topicName", + TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName()); + } + break; + case BASIC: + populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails); + + traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel()); + traceDetails.put("Consumer.subType", consumer.subType().name()); + traceDetails.put("Consumer.clientAddress", consumer.getClientAddress()); + break; + case FULL: + populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails); + populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails); + + traceDetails.put("Consumer.metadata", consumer.getMetadata()); + break; + default: + log.debug("Unknown tracing level: {}", level); + break; + } + } + + private static void populateProducerDetails( + TraceLevel level, Producer producer, Map traceDetails) { + if (producer == null) { + traceDetails.put("Producer", "null"); + return; + } + + switch (level) { + case MIMIMAL: + traceDetails.put("Producer.producerId", producer.getProducerId()); + traceDetails.put("Producer.producerName", producer.getProducerName()); + traceDetails.put("Producer.accessMode", producer.getAccessMode().name()); + if (producer.getTopic() != null) { + traceDetails.put( + "Producer.topicName", + TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); + } + break; + case BASIC: + populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails); + + traceDetails.put("Producer.clientAddress", producer.getClientAddress()); + break; + case FULL: + populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails); + populateProducerDetails(TraceLevel.BASIC, producer, traceDetails); + + traceDetails.put("Producer.metadata", producer.getMetadata()); + if (producer.getSchemaVersion() != null) { + traceDetails.put("Producer.schemaVersion", producer.getSchemaVersion().toString()); + } + traceDetails.put("producer.remoteCluster", producer.getRemoteCluster()); + break; + default: + log.debug("Unknown tracing level: {}", level); + break; + } + } + + private static void populateMessageMetadataDetails( + TraceLevel level, MessageMetadata msgMetadata, Map traceDetails) { + if (msgMetadata == null) { + traceDetails.put("MessageMetadata", "null"); + return; + } + + switch (level) { + case MIMIMAL: + traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId()); + traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName()); + traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey()); + break; + case BASIC: + populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails); + + traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize()); + traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize()); + traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); + break; + case FULL: + populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails); + populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails); + + traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime()); + traceDetails.put("MessageMetadata.eventTime", msgMetadata.getEventTime()); + traceDetails.put("MessageMetadata.replicatedFrom", msgMetadata.getReplicatedFrom()); + traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid()); + break; + default: + log.debug("Unknown tracing level: {}", level); + break; + } + } + + private static void populateEntryDetails( + TraceLevel level, Entry entry, Map traceDetails) { + if (entry == null) { + traceDetails.put("Entry", "null"); + return; + } + + switch (level) { + case MIMIMAL: + traceDetails.put("Entry.ledgerId", entry.getLedgerId()); + traceDetails.put("Entry.entryId", entry.getEntryId()); + break; + case BASIC: + populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails); + + traceDetails.put("Entry.length", entry.getLength()); + break; + case FULL: + populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails); + populateEntryDetails(TraceLevel.BASIC, entry, traceDetails); + + traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails); + break; + default: + log.debug("Unknown tracing level: {}", level); + break; + } + } + + private static void populatePublishContext( + Topic.PublishContext publishContext, Map traceDetails) { + traceDetails.put("publishContext.isMarkerMessage", publishContext.isMarkerMessage()); + traceDetails.put("publishContext.isChunked", publishContext.isChunked()); + traceDetails.put("publishContext.numberOfMessages", publishContext.getNumberOfMessages()); + + traceDetails.put("publishContext.entryTimestamp", publishContext.getEntryTimestamp()); + traceDetails.put("publishContext.msgSize", publishContext.getMsgSize()); + traceDetails.put("publishContext.producerName", publishContext.getProducerName()); + traceDetails.put( + "publishContext.originalProducerName", publishContext.getOriginalProducerName()); + traceDetails.put("publishContext.originalSequenceId", publishContext.getOriginalSequenceId()); + traceDetails.put("publishContext.sequenceId", publishContext.getSequenceId()); + } + + private static void traceByteBuf(String key, ByteBuf buf, Map traceDetails) { + if (buf == null) return; + + if (buf.readableBytes() < MAX_DATA_LENGTH) { + traceDetails.put(key, Hex.encodeHex(buf.nioBuffer())); + } else { + traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); + } + } + + private static TraceLevel getTracingLevel(Subscription sub) { + if (sub == null + || sub.getSubscriptionProperties() == null + || !sub.getSubscriptionProperties().containsKey("trace")) { + return TraceLevel.NONE; + } + try { + return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace")); + } catch (IllegalArgumentException e) { + log.debug( + "Invalid tracing level: {}. Setting to NONE for subscription {}", + sub.getSubscriptionProperties().get("trace"), + sub); + return TraceLevel.NONE; + } + } + + private static TraceLevel getTracingLevel(Producer producer) { + if (producer == null + || producer.getMetadata() == null + || !producer.getMetadata().containsKey("trace")) { + return TraceLevel.NONE; + } + try { + return TraceLevel.valueOf(producer.getMetadata().get("trace")); + } catch (IllegalArgumentException e) { + log.debug( + "Invalid tracing level: {}. Setting to NONE for producer {}", + producer.getMetadata().get("trace"), + producer); + return TraceLevel.NONE; + } + } + + // private boolean needToTraceTopic(Topic topic) { + // if (topic == null) return false; + // + // // todo: how to read topic props + // return true; + // } + + /* *************************** + ** Administrative events + ******************************/ + + public void onConnectionCreated(ServerCnx cnx) { + if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + + TraceLevel level = getTracingLevel(cnx); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + log.info("Connection created: {}", traceDetails); + } + + public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { + if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + + TraceLevel level = getTracingLevel(cnx); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + populateProducerDetails(level, producer, traceDetails); + + traceDetails.put("metadata", metadata); + + log.info("Producer created: {}", traceDetails); + } + + public void producerClosed(ServerCnx cnx, Producer producer, Map metadata) { + if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + + TraceLevel level = getTracingLevel(cnx); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + populateProducerDetails(level, producer, traceDetails); + + traceDetails.put("metadata", metadata); + + log.info("Producer closed: {}", traceDetails); + } + + public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { + if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + + TraceLevel level = getTracingLevel(cnx); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + populateConsumerDetails(level, consumer, traceDetails); + populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); + + traceDetails.put("metadata", metadata); + + log.info("Consumer created: {}", traceDetails); + } + + public void consumerClosed(ServerCnx cnx, Consumer consumer, Map metadata) { + if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + + TraceLevel level = getTracingLevel(cnx); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + populateConsumerDetails(level, consumer, traceDetails); + populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); + + traceDetails.put("metadata", metadata); + + log.info("Consumer closed: {}", traceDetails); + } + + public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { + if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + + TraceLevel level = getTracingLevel(cnx); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + traceDetails.put("command", command.toString()); + log.info("Pulsar command called: {}", traceDetails); + } + + public void onConnectionClosed(ServerCnx cnx) { + if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + + TraceLevel level = getTracingLevel(cnx); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + log.info("Connection closed: {}", traceDetails); + } + + /* *************************** + ** Message events + ******************************/ + + public void beforeSendMessage( + Subscription subscription, + Entry entry, + long[] ackSet, + MessageMetadata msgMetadata, + Consumer consumer) { + if (!getEnabledEvents(consumer.cnx().getBrokerService().getPulsar()) + .contains(EventReasons.MESSAGE)) return; + + TraceLevel level = getTracingLevel(subscription); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConsumerDetails(level, consumer, traceDetails); + populateSubscriptionDetails(level, subscription, traceDetails); + populateEntryDetails(level, entry, traceDetails); + populateMessageMetadataDetails(level, msgMetadata, traceDetails); + + log.info("Before sending message: {}", traceDetails); + } + + public void onMessagePublish( + Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) { + + if (!getEnabledEvents(producer.getCnx().getBrokerService().getPulsar()) + .contains(EventReasons.MESSAGE)) return; + + TraceLevel level = getTracingLevel(producer); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateProducerDetails(level, producer, traceDetails); + + traceByteBuf("headersAndPayload", headersAndPayload, traceDetails); + + populatePublishContext(publishContext, traceDetails); + + log.info("Message publish: {}", traceDetails); + } + + public void messageProduced( + ServerCnx cnx, + Producer producer, + long startTimeNs, + long ledgerId, + long entryId, + Topic.PublishContext publishContext) { + if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return; + + TraceLevel level = getTracingLevel(producer); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + + populateConnectionDetails(level, cnx, traceDetails); + populateProducerDetails(level, producer, traceDetails); + populatePublishContext(publishContext, traceDetails); + + traceDetails.put("ledgerId", ledgerId); + traceDetails.put("entryId", entryId); + traceDetails.put("startTimeNs", startTimeNs); + + log.info("Message produced: {}", traceDetails); + } + + public void messageDispatched( + ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) { + if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return; + + TraceLevel level = getTracingLevel(consumer.getSubscription()); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + populateConsumerDetails(level, consumer, traceDetails); + populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); + + traceDetails.put("ledgerId", ledgerId); + traceDetails.put("entryId", entryId); + + traceByteBuf("headersAndPayload", headersAndPayload, traceDetails); + + log.info("After dispatching message: {}", traceDetails); + } + + public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { + if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return; + + TraceLevel level = getTracingLevel(consumer.getSubscription()); + if (level == TraceLevel.NONE) return; + + Map traceDetails = new TreeMap<>(); + populateConnectionDetails(level, cnx, traceDetails); + populateConsumerDetails(level, consumer, traceDetails); + populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); + + traceDetails.put("ack.type", ackCmd.getAckType().name()); + traceDetails.put("ack.consumerId", ackCmd.getConsumerId()); + traceDetails.put( + "ack.messageIds", + ackCmd + .getMessageIdsList() + .stream() + .map(x -> x.getLedgerId() + ":" + x.getEntryId()) + .collect(Collectors.toList())); + + log.info("Message acked: {}", traceDetails); + } + + /* *************************** + ** Transaction events + ******************************/ + + public void txnOpened(long tcId, String txnID) { + // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) { + Map traceDetails = new TreeMap<>(); + traceDetails.put("tcId", tcId); + traceDetails.put("txnID", txnID); + + log.info("Transaction opened: {}", traceDetails); + // } + } + + public void txnEnded(String txnID, long txnAction) { + // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) { + Map traceDetails = new TreeMap<>(); + traceDetails.put("txnID", txnID); + traceDetails.put("txnAction", txnAction); + + log.info("Transaction closed: {}", traceDetails); + // } + } + + /* *************************** + ** Servlet events + ******************************/ + + public void onWebserviceRequest(ServletRequest request) + throws IOException, ServletException, InterceptException { + // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) { + // log.info("onWebserviceRequest: Tracing servlet requests not supported"); + // } + } + + public void onWebserviceResponse(ServletRequest request, ServletResponse response) + throws IOException, ServletException { + // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) { + // log.info("onWebserviceResponse: Tracing servlet requests not supported"); + // } + } + + // not needed + // public void onFilter(ServletRequest request, ServletResponse response, FilterChain chain) +} diff --git a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml new file mode 100644 index 00000000..38748bb9 --- /dev/null +++ b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml @@ -0,0 +1,3 @@ +interceptorClass: com.datastax.oss.pulsar.tracing.BrokerTracing +name: pulsar-tracing +description: Starlight for JMS - support for server side tracing \ No newline at end of file diff --git a/pulsar-tracing/src/test/resources/log4j2.xml b/pulsar-tracing/src/test/resources/log4j2.xml new file mode 100644 index 00000000..485f6705 --- /dev/null +++ b/pulsar-tracing/src/test/resources/log4j2.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + From 91e51864fd52f409366310923ada743e71833bdd Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Fri, 12 Apr 2024 11:12:41 -0700 Subject: [PATCH 02/29] various renamings and tweaks --- pom.xml | 2 +- .../pom.xml | 4 +- .../pulsar/jms}/tracing/BrokerTracing.java | 91 ++++++++++--------- .../META-INF/services/broker_interceptor.yml | 3 + .../src/test/resources/log4j2.xml | 0 .../META-INF/services/broker_interceptor.yml | 3 - 6 files changed, 53 insertions(+), 50 deletions(-) rename {pulsar-tracing => pulsar-jms-tracing}/pom.xml (95%) rename {pulsar-tracing/src/main/java/com/datastax/oss/pulsar => pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms}/tracing/BrokerTracing.java (89%) create mode 100644 pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml rename {pulsar-tracing => pulsar-jms-tracing}/src/test/resources/log4j2.xml (100%) delete mode 100644 pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml diff --git a/pom.xml b/pom.xml index 8f9c2802..a24e298a 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ activemq-filters pulsar-jms-filters - pulsar-tracing + pulsar-jms-tracing pulsar-jms-admin-api pulsar-jms resource-adapter diff --git a/pulsar-tracing/pom.xml b/pulsar-jms-tracing/pom.xml similarity index 95% rename from pulsar-tracing/pom.xml rename to pulsar-jms-tracing/pom.xml index e4a65161..2d4536d8 100644 --- a/pulsar-tracing/pom.xml +++ b/pulsar-jms-tracing/pom.xml @@ -23,7 +23,7 @@ 4.1.1-alpha-SNAPSHOT 4.0.0 - pulsar-tracing + pulsar-jms-tracing jar DataStax Starlight for JMS - Broker Side Filters @@ -64,7 +64,7 @@ 1.3.2 true - pulsar-tracing-${project.version} + pulsar-jms-tracing-${project.version} nar diff --git a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java similarity index 89% rename from pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java rename to pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index e18e2cec..82e6ec3b 100644 --- a/pulsar-tracing/src/main/java/com/datastax/oss/pulsar/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.datastax.oss.pulsar.tracing; +package com.datastax.oss.pulsar.jms.tracing; import io.netty.buffer.ByteBuf; import java.io.IOException; @@ -44,6 +44,8 @@ @Slf4j public class BrokerTracing implements BrokerInterceptor { + public static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing"); + static final int MAX_DATA_LENGTH = 1024; public enum EventReasons { @@ -55,7 +57,7 @@ public enum EventReasons { public enum TraceLevel { NONE, - MIMIMAL, + MINIMAL, BASIC, FULL } @@ -79,7 +81,8 @@ private Set getEnabledEvents(ServerCnx cnx) { private Set getEnabledEvents(PulsarService pulsar) { // todo: do this less frequently - String events = pulsar.getConfiguration().getProperties().getProperty("tracing_event_list", ""); + String events = + pulsar.getConfiguration().getProperties().getProperty("jmsTracingEventList", ""); Set enabledEvents = new HashSet<>(); @@ -101,7 +104,7 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) { .getPulsar() .getConfiguration() .getProperties() - .getProperty("tracing_level", defaultTraceLevel.toString()); + .getProperty("jmsTracingLevel", defaultTraceLevel.toString()); try { return TraceLevel.valueOf(level); } catch (IllegalArgumentException e) { @@ -118,17 +121,17 @@ private static void populateConnectionDetails( } switch (level) { - case MIMIMAL: + case MINIMAL: traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString()); break; case BASIC: - populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails); + populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion()); traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); break; case FULL: - populateConnectionDetails(TraceLevel.MIMIMAL, cnx, traceDetails); + populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); traceDetails.put("ServerCnx.authRole", cnx.getAuthRole()); @@ -137,7 +140,7 @@ private static void populateConnectionDetails( "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); break; default: - log.debug("Unknown tracing level: {}", level); + log.warn("Unknown tracing level: {}", level); break; } } @@ -150,14 +153,14 @@ private static void populateSubscriptionDetails( } switch (level) { - case MIMIMAL: + case MINIMAL: traceDetails.put("Subscription.name", sub.getName()); traceDetails.put( "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); traceDetails.put("Subscription.type", sub.getType().name()); break; case BASIC: - populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails); + populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); if (sub.getConsumers() != null) { traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size()); @@ -168,13 +171,13 @@ private static void populateSubscriptionDetails( break; case FULL: - populateSubscriptionDetails(TraceLevel.MIMIMAL, sub, traceDetails); + populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails); traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties()); break; default: - log.debug("Unknown tracing level: {}", level); + log.warn("Unknown tracing level: {}", level); break; } } @@ -187,7 +190,7 @@ private static void populateConsumerDetails( } switch (level) { - case MIMIMAL: + case MINIMAL: traceDetails.put("Consumer.name", consumer.consumerName()); traceDetails.put("Consumer.consumerId", consumer.consumerId()); if (consumer.getSubscription() != null) { @@ -198,20 +201,20 @@ private static void populateConsumerDetails( } break; case BASIC: - populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails); + populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel()); traceDetails.put("Consumer.subType", consumer.subType().name()); traceDetails.put("Consumer.clientAddress", consumer.getClientAddress()); break; case FULL: - populateConsumerDetails(TraceLevel.MIMIMAL, consumer, traceDetails); + populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails); traceDetails.put("Consumer.metadata", consumer.getMetadata()); break; default: - log.debug("Unknown tracing level: {}", level); + log.warn("Unknown tracing level: {}", level); break; } } @@ -224,7 +227,7 @@ private static void populateProducerDetails( } switch (level) { - case MIMIMAL: + case MINIMAL: traceDetails.put("Producer.producerId", producer.getProducerId()); traceDetails.put("Producer.producerName", producer.getProducerName()); traceDetails.put("Producer.accessMode", producer.getAccessMode().name()); @@ -235,12 +238,12 @@ private static void populateProducerDetails( } break; case BASIC: - populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails); + populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); traceDetails.put("Producer.clientAddress", producer.getClientAddress()); break; case FULL: - populateProducerDetails(TraceLevel.MIMIMAL, producer, traceDetails); + populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); populateProducerDetails(TraceLevel.BASIC, producer, traceDetails); traceDetails.put("Producer.metadata", producer.getMetadata()); @@ -250,7 +253,7 @@ private static void populateProducerDetails( traceDetails.put("producer.remoteCluster", producer.getRemoteCluster()); break; default: - log.debug("Unknown tracing level: {}", level); + log.warn("Unknown tracing level: {}", level); break; } } @@ -263,20 +266,20 @@ private static void populateMessageMetadataDetails( } switch (level) { - case MIMIMAL: + case MINIMAL: traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId()); traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName()); traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey()); break; case BASIC: - populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails); + populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize()); traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize()); traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); break; case FULL: - populateMessageMetadataDetails(TraceLevel.MIMIMAL, msgMetadata, traceDetails); + populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails); traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime()); @@ -285,7 +288,7 @@ private static void populateMessageMetadataDetails( traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid()); break; default: - log.debug("Unknown tracing level: {}", level); + log.warn("Unknown tracing level: {}", level); break; } } @@ -298,23 +301,23 @@ private static void populateEntryDetails( } switch (level) { - case MIMIMAL: + case MINIMAL: traceDetails.put("Entry.ledgerId", entry.getLedgerId()); traceDetails.put("Entry.entryId", entry.getEntryId()); break; case BASIC: - populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails); + populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); traceDetails.put("Entry.length", entry.getLength()); break; case FULL: - populateEntryDetails(TraceLevel.MIMIMAL, entry, traceDetails); + populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); populateEntryDetails(TraceLevel.BASIC, entry, traceDetails); traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails); break; default: - log.debug("Unknown tracing level: {}", level); + log.warn("Unknown tracing level: {}", level); break; } } @@ -353,7 +356,7 @@ private static TraceLevel getTracingLevel(Subscription sub) { try { return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace")); } catch (IllegalArgumentException e) { - log.debug( + log.warn( "Invalid tracing level: {}. Setting to NONE for subscription {}", sub.getSubscriptionProperties().get("trace"), sub); @@ -370,7 +373,7 @@ private static TraceLevel getTracingLevel(Producer producer) { try { return TraceLevel.valueOf(producer.getMetadata().get("trace")); } catch (IllegalArgumentException e) { - log.debug( + log.warn( "Invalid tracing level: {}. Setting to NONE for producer {}", producer.getMetadata().get("trace"), producer); @@ -397,7 +400,7 @@ public void onConnectionCreated(ServerCnx cnx) { Map traceDetails = new TreeMap<>(); populateConnectionDetails(level, cnx, traceDetails); - log.info("Connection created: {}", traceDetails); + tracer.info("Connection created: {}", traceDetails); } public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { @@ -412,7 +415,7 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { @@ -427,7 +430,7 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map traceDetails.put("metadata", metadata); - log.info("Producer closed: {}", traceDetails); + tracer.info("Producer closed: {}", traceDetails); } public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { @@ -443,7 +446,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { @@ -459,7 +462,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map traceDetails.put("metadata", metadata); - log.info("Consumer closed: {}", traceDetails); + tracer.info("Consumer closed: {}", traceDetails); } public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { @@ -471,7 +474,7 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept Map traceDetails = new TreeMap<>(); populateConnectionDetails(level, cnx, traceDetails); traceDetails.put("command", command.toString()); - log.info("Pulsar command called: {}", traceDetails); + tracer.info("Pulsar command called: {}", traceDetails); } public void onConnectionClosed(ServerCnx cnx) { @@ -482,7 +485,7 @@ public void onConnectionClosed(ServerCnx cnx) { Map traceDetails = new TreeMap<>(); populateConnectionDetails(level, cnx, traceDetails); - log.info("Connection closed: {}", traceDetails); + tracer.info("Connection closed: {}", traceDetails); } /* *************************** @@ -507,7 +510,7 @@ public void beforeSendMessage( populateEntryDetails(level, entry, traceDetails); populateMessageMetadataDetails(level, msgMetadata, traceDetails); - log.info("Before sending message: {}", traceDetails); + tracer.info("Before sending message: {}", traceDetails); } public void onMessagePublish( @@ -526,7 +529,7 @@ public void onMessagePublish( populatePublishContext(publishContext, traceDetails); - log.info("Message publish: {}", traceDetails); + tracer.info("Message publish: {}", traceDetails); } public void messageProduced( @@ -551,7 +554,7 @@ public void messageProduced( traceDetails.put("entryId", entryId); traceDetails.put("startTimeNs", startTimeNs); - log.info("Message produced: {}", traceDetails); + tracer.info("Message produced: {}", traceDetails); } public void messageDispatched( @@ -571,7 +574,7 @@ public void messageDispatched( traceByteBuf("headersAndPayload", headersAndPayload, traceDetails); - log.info("After dispatching message: {}", traceDetails); + tracer.info("After dispatching message: {}", traceDetails); } public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { @@ -595,7 +598,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { .map(x -> x.getLedgerId() + ":" + x.getEntryId()) .collect(Collectors.toList())); - log.info("Message acked: {}", traceDetails); + tracer.info("Message acked: {}", traceDetails); } /* *************************** @@ -608,7 +611,7 @@ public void txnOpened(long tcId, String txnID) { traceDetails.put("tcId", tcId); traceDetails.put("txnID", txnID); - log.info("Transaction opened: {}", traceDetails); + tracer.info("Transaction opened: {}", traceDetails); // } } @@ -618,7 +621,7 @@ public void txnEnded(String txnID, long txnAction) { traceDetails.put("txnID", txnID); traceDetails.put("txnAction", txnAction); - log.info("Transaction closed: {}", traceDetails); + tracer.info("Transaction closed: {}", traceDetails); // } } diff --git a/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml new file mode 100644 index 00000000..a47fa05f --- /dev/null +++ b/pulsar-jms-tracing/src/main/resources/META-INF/services/broker_interceptor.yml @@ -0,0 +1,3 @@ +interceptorClass: com.datastax.oss.pulsar.jms.tracing.BrokerTracing +name: jms-tracing +description: Starlight for JMS - support for server side tracing \ No newline at end of file diff --git a/pulsar-tracing/src/test/resources/log4j2.xml b/pulsar-jms-tracing/src/test/resources/log4j2.xml similarity index 100% rename from pulsar-tracing/src/test/resources/log4j2.xml rename to pulsar-jms-tracing/src/test/resources/log4j2.xml diff --git a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml b/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml deleted file mode 100644 index 38748bb9..00000000 --- a/pulsar-tracing/src/main/resources/META-INF/services/broker_interceptor.yml +++ /dev/null @@ -1,3 +0,0 @@ -interceptorClass: com.datastax.oss.pulsar.tracing.BrokerTracing -name: pulsar-tracing -description: Starlight for JMS - support for server side tracing \ No newline at end of file From ea914204b8653be053ae27499e33f6e022627211 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Fri, 12 Apr 2024 12:16:28 -0700 Subject: [PATCH 03/29] traces as json --- pulsar-jms-tracing/pom.xml | 4 + .../oss/pulsar/jms/tracing/BrokerTracing.java | 315 +++++++++++------- 2 files changed, 200 insertions(+), 119 deletions(-) diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml index 2d4536d8..c60c0b4a 100644 --- a/pulsar-jms-tracing/pom.xml +++ b/pulsar-jms-tracing/pom.xml @@ -30,6 +30,10 @@ ${project.build.directory} + + com.fasterxml.jackson.core + jackson-databind + org.projectlombok lombok diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 82e6ec3b..dd41a4c4 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -15,6 +15,9 @@ */ package com.datastax.oss.pulsar.jms.tracing; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import io.netty.buffer.ByteBuf; import java.io.IOException; import java.util.HashSet; @@ -44,7 +47,12 @@ @Slf4j public class BrokerTracing implements BrokerInterceptor { - public static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing"); + private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing"); + + private static final ObjectMapper mapper = + new ObjectMapper() + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) + .enable(SerializationFeature.WRITE_NULL_MAP_VALUES); static final int MAX_DATA_LENGTH = 1024; @@ -65,6 +73,19 @@ public enum TraceLevel { // private final Set defaultEnabledEvents = new HashSet<>(); private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC; + public static void trace(String message, Map traceDetails) { + Map trace = new TreeMap<>(); + trace.put("message", message); + trace.put("details", traceDetails); + + try { + String loggableJsonString = mapper.writeValueAsString(trace); + tracer.info(loggableJsonString); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + public void initialize(PulsarService pulsarService) { // defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE); // defaultEnabledEvents.add(EventReasons.MESSAGE); @@ -113,31 +134,39 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) { } } + public static Map getConnectionDetails(TraceLevel level, ServerCnx cnx) { + if (cnx == null) { + return null; + } + + Map details = new TreeMap<>(); + populateConnectionDetails(level, cnx, details); + return details; + } + private static void populateConnectionDetails( TraceLevel level, ServerCnx cnx, Map traceDetails) { if (cnx == null) { - traceDetails.put("ServerCnx", "null"); return; } switch (level) { case MINIMAL: - traceDetails.put("ServerCnx.clientAddress", cnx.clientAddress().toString()); + traceDetails.put("clientAddress", cnx.clientAddress().toString()); break; case BASIC: populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); - traceDetails.put("ServerCnx.clientVersion", cnx.getClientVersion()); - traceDetails.put("ServerCnx.clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); + traceDetails.put("clientVersion", cnx.getClientVersion()); + traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); break; case FULL: populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); - traceDetails.put("ServerCnx.authRole", cnx.getAuthRole()); - traceDetails.put("ServerCnx.authMethod", cnx.getAuthMethod()); - traceDetails.put( - "ServerCnx.authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); + traceDetails.put("authRole", cnx.getAuthRole()); + traceDetails.put("authMethod", cnx.getAuthMethod()); + traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); break; default: log.warn("Unknown tracing level: {}", level); @@ -145,27 +174,35 @@ private static void populateConnectionDetails( } } + public static Map getSubscriptionDetails(TraceLevel level, Subscription sub) { + if (sub == null) { + return null; + } + + Map details = new TreeMap<>(); + populateSubscriptionDetails(level, sub, details); + return details; + } + private static void populateSubscriptionDetails( TraceLevel level, Subscription sub, Map traceDetails) { if (sub == null) { - traceDetails.put("Subscription", "null"); return; } switch (level) { case MINIMAL: - traceDetails.put("Subscription.name", sub.getName()); - traceDetails.put( - "Subscription.topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); - traceDetails.put("Subscription.type", sub.getType().name()); + traceDetails.put("name", sub.getName()); + traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); + traceDetails.put("type", sub.getType().name()); break; case BASIC: populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); if (sub.getConsumers() != null) { - traceDetails.put("Subscription.numberOfConsumers", sub.getConsumers().size()); + traceDetails.put("numberOfConsumers", sub.getConsumers().size()); traceDetails.put( - "Subscription.namesOfConsumers", + "namesOfConsumers", sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList())); } @@ -174,7 +211,7 @@ private static void populateSubscriptionDetails( populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails); - traceDetails.put("Subscription.subscriptionProperties", sub.getSubscriptionProperties()); + traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); break; default: log.warn("Unknown tracing level: {}", level); @@ -182,36 +219,45 @@ private static void populateSubscriptionDetails( } } + public static Map getConsumerDetails(TraceLevel level, Consumer consumer) { + if (consumer == null) { + return null; + } + + Map details = new TreeMap<>(); + populateConsumerDetails(level, consumer, details); + return details; + } + private static void populateConsumerDetails( TraceLevel level, Consumer consumer, Map traceDetails) { if (consumer == null) { - traceDetails.put("Consumer", "null"); return; } switch (level) { case MINIMAL: - traceDetails.put("Consumer.name", consumer.consumerName()); - traceDetails.put("Consumer.consumerId", consumer.consumerId()); + traceDetails.put("name", consumer.consumerName()); + traceDetails.put("consumerId", consumer.consumerId()); if (consumer.getSubscription() != null) { - traceDetails.put("Consumer.subscriptionName", consumer.getSubscription().getName()); + traceDetails.put("subscriptionName", consumer.getSubscription().getName()); traceDetails.put( - "Consumer.topicName", + "topicName", TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName()); } break; case BASIC: populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); - traceDetails.put("Consumer.priorityLevel", consumer.getPriorityLevel()); - traceDetails.put("Consumer.subType", consumer.subType().name()); - traceDetails.put("Consumer.clientAddress", consumer.getClientAddress()); + traceDetails.put("priorityLevel", consumer.getPriorityLevel()); + traceDetails.put("subType", consumer.subType().name()); + traceDetails.put("clientAddress", consumer.getClientAddress()); break; case FULL: populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails); - traceDetails.put("Consumer.metadata", consumer.getMetadata()); + traceDetails.put("metadata", consumer.getMetadata()); break; default: log.warn("Unknown tracing level: {}", level); @@ -219,38 +265,46 @@ private static void populateConsumerDetails( } } + public static Map getProducerDetails(TraceLevel level, Producer producer) { + if (producer == null) { + return null; + } + + Map details = new TreeMap<>(); + populateProducerDetails(level, producer, details); + return details; + } + private static void populateProducerDetails( TraceLevel level, Producer producer, Map traceDetails) { if (producer == null) { - traceDetails.put("Producer", "null"); return; } switch (level) { case MINIMAL: - traceDetails.put("Producer.producerId", producer.getProducerId()); - traceDetails.put("Producer.producerName", producer.getProducerName()); - traceDetails.put("Producer.accessMode", producer.getAccessMode().name()); + traceDetails.put("producerId", producer.getProducerId()); + traceDetails.put("producerName", producer.getProducerName()); + traceDetails.put("accessMode", producer.getAccessMode().name()); if (producer.getTopic() != null) { traceDetails.put( - "Producer.topicName", - TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); + "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); } break; case BASIC: populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); - traceDetails.put("Producer.clientAddress", producer.getClientAddress()); + traceDetails.put("clientAddress", producer.getClientAddress()); break; case FULL: populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); populateProducerDetails(TraceLevel.BASIC, producer, traceDetails); - traceDetails.put("Producer.metadata", producer.getMetadata()); + traceDetails.put("metadata", producer.getMetadata()); if (producer.getSchemaVersion() != null) { - traceDetails.put("Producer.schemaVersion", producer.getSchemaVersion().toString()); + traceDetails.put("schemaVersion", producer.getSchemaVersion().toString()); } - traceDetails.put("producer.remoteCluster", producer.getRemoteCluster()); + traceDetails.put("remoteCluster", producer.getRemoteCluster()); break; default: log.warn("Unknown tracing level: {}", level); @@ -258,34 +312,44 @@ private static void populateProducerDetails( } } + public static Map getMessageMetadataDetails( + TraceLevel level, MessageMetadata msgMetadata) { + if (msgMetadata == null) { + return null; + } + + Map details = new TreeMap<>(); + populateMessageMetadataDetails(level, msgMetadata, details); + return details; + } + private static void populateMessageMetadataDetails( TraceLevel level, MessageMetadata msgMetadata, Map traceDetails) { if (msgMetadata == null) { - traceDetails.put("MessageMetadata", "null"); return; } switch (level) { case MINIMAL: - traceDetails.put("MessageMetadata.sequenceId", msgMetadata.getSequenceId()); - traceDetails.put("MessageMetadata.producerName", msgMetadata.getProducerName()); - traceDetails.put("MessageMetadata.partitionKey", msgMetadata.getPartitionKey()); + traceDetails.put("sequenceId", msgMetadata.getSequenceId()); + traceDetails.put("producerName", msgMetadata.getProducerName()); + traceDetails.put("partitionKey", msgMetadata.getPartitionKey()); break; case BASIC: populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); - traceDetails.put("MessageMetadata.uncompressedSize", msgMetadata.getUncompressedSize()); - traceDetails.put("MessageMetadata.serializedSize", msgMetadata.getSerializedSize()); - traceDetails.put("MessageMetadata.numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); + traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); + traceDetails.put("serializedSize", msgMetadata.getSerializedSize()); + traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); break; case FULL: populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails); - traceDetails.put("MessageMetadata.publishTime", msgMetadata.getPublishTime()); - traceDetails.put("MessageMetadata.eventTime", msgMetadata.getEventTime()); - traceDetails.put("MessageMetadata.replicatedFrom", msgMetadata.getReplicatedFrom()); - traceDetails.put("MessageMetadata.uuid", msgMetadata.getUuid()); + traceDetails.put("publishTime", msgMetadata.getPublishTime()); + traceDetails.put("eventTime", msgMetadata.getEventTime()); + traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom()); + traceDetails.put("uuid", msgMetadata.getUuid()); break; default: log.warn("Unknown tracing level: {}", level); @@ -293,28 +357,37 @@ private static void populateMessageMetadataDetails( } } + public static Map getEntryDetails(TraceLevel level, Entry entry) { + if (entry == null) { + return null; + } + + Map details = new TreeMap<>(); + populateEntryDetails(level, entry, details); + return details; + } + private static void populateEntryDetails( TraceLevel level, Entry entry, Map traceDetails) { if (entry == null) { - traceDetails.put("Entry", "null"); return; } switch (level) { case MINIMAL: - traceDetails.put("Entry.ledgerId", entry.getLedgerId()); - traceDetails.put("Entry.entryId", entry.getEntryId()); + traceDetails.put("ledgerId", entry.getLedgerId()); + traceDetails.put("entryId", entry.getEntryId()); break; case BASIC: populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); - traceDetails.put("Entry.length", entry.getLength()); + traceDetails.put("length", entry.getLength()); break; case FULL: populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); populateEntryDetails(TraceLevel.BASIC, entry, traceDetails); - traceByteBuf("Entry.data", entry.getDataBuffer(), traceDetails); + traceByteBuf("data", entry.getDataBuffer(), traceDetails); break; default: log.warn("Unknown tracing level: {}", level); @@ -322,19 +395,28 @@ private static void populateEntryDetails( } } + public static Map getPublishContextDetails(Topic.PublishContext publishContext) { + if (publishContext == null) { + return null; + } + + Map details = new TreeMap<>(); + populatePublishContext(publishContext, details); + return details; + } + private static void populatePublishContext( Topic.PublishContext publishContext, Map traceDetails) { - traceDetails.put("publishContext.isMarkerMessage", publishContext.isMarkerMessage()); - traceDetails.put("publishContext.isChunked", publishContext.isChunked()); - traceDetails.put("publishContext.numberOfMessages", publishContext.getNumberOfMessages()); + traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage()); + traceDetails.put("isChunked", publishContext.isChunked()); + traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages()); - traceDetails.put("publishContext.entryTimestamp", publishContext.getEntryTimestamp()); - traceDetails.put("publishContext.msgSize", publishContext.getMsgSize()); - traceDetails.put("publishContext.producerName", publishContext.getProducerName()); - traceDetails.put( - "publishContext.originalProducerName", publishContext.getOriginalProducerName()); - traceDetails.put("publishContext.originalSequenceId", publishContext.getOriginalSequenceId()); - traceDetails.put("publishContext.sequenceId", publishContext.getSequenceId()); + traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp()); + traceDetails.put("msgSize", publishContext.getMsgSize()); + traceDetails.put("producerName", publishContext.getProducerName()); + traceDetails.put("originalProducerName", publishContext.getOriginalProducerName()); + traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId()); + traceDetails.put("sequenceId", publishContext.getSequenceId()); } private static void traceByteBuf(String key, ByteBuf buf, Map traceDetails) { @@ -399,8 +481,8 @@ public void onConnectionCreated(ServerCnx cnx) { if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - tracer.info("Connection created: {}", traceDetails); + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + trace("Connection created", traceDetails); } public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { @@ -410,12 +492,11 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - populateProducerDetails(level, producer, traceDetails); - + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("producer", getProducerDetails(level, producer)); traceDetails.put("metadata", metadata); - tracer.info("Producer created: {}", traceDetails); + trace("Producer created", traceDetails); } public void producerClosed(ServerCnx cnx, Producer producer, Map metadata) { @@ -425,12 +506,11 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - populateProducerDetails(level, producer, traceDetails); - + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("producer", getProducerDetails(level, producer)); traceDetails.put("metadata", metadata); - tracer.info("Producer closed: {}", traceDetails); + trace("Producer closed", traceDetails); } public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { @@ -440,13 +520,12 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - populateConsumerDetails(level, consumer, traceDetails); - populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); - + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("consumer", getConsumerDetails(level, consumer)); + traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); traceDetails.put("metadata", metadata); - tracer.info("Consumer created: {}", traceDetails); + trace("Consumer created", traceDetails); } public void consumerClosed(ServerCnx cnx, Consumer consumer, Map metadata) { @@ -456,13 +535,12 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - populateConsumerDetails(level, consumer, traceDetails); - populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); - + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("consumer", getConsumerDetails(level, consumer)); + traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); traceDetails.put("metadata", metadata); - tracer.info("Consumer closed: {}", traceDetails); + trace("Consumer closed", traceDetails); } public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { @@ -472,9 +550,10 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); traceDetails.put("command", command.toString()); - tracer.info("Pulsar command called: {}", traceDetails); + + trace("Pulsar command called", traceDetails); } public void onConnectionClosed(ServerCnx cnx) { @@ -484,8 +563,9 @@ public void onConnectionClosed(ServerCnx cnx) { if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - tracer.info("Connection closed: {}", traceDetails); + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + + trace("Connection closed", traceDetails); } /* *************************** @@ -505,12 +585,12 @@ public void beforeSendMessage( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConsumerDetails(level, consumer, traceDetails); - populateSubscriptionDetails(level, subscription, traceDetails); - populateEntryDetails(level, entry, traceDetails); - populateMessageMetadataDetails(level, msgMetadata, traceDetails); + traceDetails.put("subscription", getSubscriptionDetails(level, subscription)); + traceDetails.put("consumer", getConsumerDetails(level, consumer)); + traceDetails.put("entry", getEntryDetails(level, entry)); + traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata)); - tracer.info("Before sending message: {}", traceDetails); + trace("Before sending message", traceDetails); } public void onMessagePublish( @@ -523,13 +603,11 @@ public void onMessagePublish( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateProducerDetails(level, producer, traceDetails); - + traceDetails.put("producer", getProducerDetails(level, producer)); + traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceByteBuf("headersAndPayload", headersAndPayload, traceDetails); - populatePublishContext(publishContext, traceDetails); - - tracer.info("Message publish: {}", traceDetails); + trace("Message publish", traceDetails); } public void messageProduced( @@ -545,16 +623,14 @@ public void messageProduced( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - - populateConnectionDetails(level, cnx, traceDetails); - populateProducerDetails(level, producer, traceDetails); - populatePublishContext(publishContext, traceDetails); - + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("producer", getProducerDetails(level, producer)); + traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceDetails.put("ledgerId", ledgerId); traceDetails.put("entryId", entryId); traceDetails.put("startTimeNs", startTimeNs); - tracer.info("Message produced: {}", traceDetails); + trace("Message produced", traceDetails); } public void messageDispatched( @@ -565,16 +641,14 @@ public void messageDispatched( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - populateConsumerDetails(level, consumer, traceDetails); - populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); - + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("consumer", getConsumerDetails(level, consumer)); + traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); traceDetails.put("ledgerId", ledgerId); traceDetails.put("entryId", entryId); - traceByteBuf("headersAndPayload", headersAndPayload, traceDetails); - tracer.info("After dispatching message: {}", traceDetails); + trace("After dispatching message", traceDetails); } public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { @@ -584,21 +658,24 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - populateConnectionDetails(level, cnx, traceDetails); - populateConsumerDetails(level, consumer, traceDetails); - populateSubscriptionDetails(level, consumer.getSubscription(), traceDetails); - - traceDetails.put("ack.type", ackCmd.getAckType().name()); - traceDetails.put("ack.consumerId", ackCmd.getConsumerId()); - traceDetails.put( - "ack.messageIds", + traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("consumer", getConsumerDetails(level, consumer)); + traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); + + Map ackDetails = new TreeMap<>(); + ackDetails.put("type", ackCmd.getAckType().name()); + ackDetails.put("consumerId", ackCmd.getConsumerId()); + ackDetails.put( + "messageIds", ackCmd .getMessageIdsList() .stream() .map(x -> x.getLedgerId() + ":" + x.getEntryId()) .collect(Collectors.toList())); - tracer.info("Message acked: {}", traceDetails); + traceDetails.put("ack", ackDetails); + + trace("Message acked", traceDetails); } /* *************************** @@ -611,7 +688,7 @@ public void txnOpened(long tcId, String txnID) { traceDetails.put("tcId", tcId); traceDetails.put("txnID", txnID); - tracer.info("Transaction opened: {}", traceDetails); + trace("Transaction opened", traceDetails); // } } @@ -621,7 +698,7 @@ public void txnEnded(String txnID, long txnAction) { traceDetails.put("txnID", txnID); traceDetails.put("txnAction", txnAction); - tracer.info("Transaction closed: {}", traceDetails); + trace("Transaction closed", traceDetails); // } } From 40f47848b7de41987ffc56df93e14dd149c0a886 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Fri, 12 Apr 2024 15:58:49 -0700 Subject: [PATCH 04/29] refactoring, cache some of configuration --- pom.xml | 6 + pulsar-jms-tracing/pom.xml | 4 + .../oss/pulsar/jms/tracing/BrokerTracing.java | 436 ++++-------------- .../oss/pulsar/jms/tracing/TracingUtils.java | 361 +++++++++++++++ 4 files changed, 449 insertions(+), 358 deletions(-) create mode 100644 pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java diff --git a/pom.xml b/pom.xml index a24e298a..0b10c2b1 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ 3.1.0 2.14.2 2.8.9 + 32.1.2-jre 1.21 4.0.3 3.1.0 @@ -107,6 +108,11 @@ slf4j-simple ${slf4j.version} + + com.google.guava + guava + ${guava.version} + org.junit.jupiter junit-jupiter-engine diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml index c60c0b4a..da7d2dda 100644 --- a/pulsar-jms-tracing/pom.xml +++ b/pulsar-jms-tracing/pom.xml @@ -34,6 +34,10 @@ com.fasterxml.jackson.core jackson-databind + + com.google.guava + guava + org.projectlombok lombok diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index dd41a4c4..d3b7f248 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -15,22 +15,33 @@ */ package com.datastax.oss.pulsar.jms.tracing; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getMessageMetadataDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import io.netty.buffer.ByteBuf; import java.io.IOException; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; -import org.apache.commons.codec.binary.Hex; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.service.Consumer; @@ -42,19 +53,60 @@ import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.intercept.InterceptException; -import org.apache.pulsar.common.naming.TopicName; @Slf4j public class BrokerTracing implements BrokerInterceptor { - private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing"); - - private static final ObjectMapper mapper = - new ObjectMapper() - .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) - .enable(SerializationFeature.WRITE_NULL_MAP_VALUES); - - static final int MAX_DATA_LENGTH = 1024; + private static final LoadingCache> jmsTracingEventList = + CacheBuilder.newBuilder() + .maximumSize(10) + .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES) + .build( + new CacheLoader>() { + public Set load(PulsarService pulsar) { + // dynamic config can get reloaded without service restart + String events = + pulsar + .getConfiguration() + .getProperties() + .getProperty("jmsTracingEventList", ""); + log.debug("read jmsTracingEventList: {}", events); + + Set enabledEvents = new HashSet<>(); + + for (String event : events.split(",")) { + try { + enabledEvents.add(EventReasons.valueOf(event.trim())); + } catch (IllegalArgumentException e) { + log.warn("Invalid event: {}. Skipping", event); + } + } + + return enabledEvents; + } + }); + + private static final LoadingCache traceLevelForService = + CacheBuilder.newBuilder() + .maximumSize(10) + .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES) + .build( + new CacheLoader() { + public TraceLevel load(PulsarService pulsar) { + String level = + pulsar + .getConfiguration() + .getProperties() + .getProperty("jmsTracingLevel", defaultTraceLevel.toString()); + try { + return TraceLevel.valueOf(level); + } catch (IllegalArgumentException e) { + log.warn( + "Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel); + return defaultTraceLevel; + } + } + }); public enum EventReasons { ADMINISTRATIVE, @@ -63,372 +115,40 @@ public enum EventReasons { SERVLET, } - public enum TraceLevel { - NONE, - MINIMAL, - BASIC, - FULL - } - - // private final Set defaultEnabledEvents = new HashSet<>(); private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC; - public static void trace(String message, Map traceDetails) { - Map trace = new TreeMap<>(); - trace.put("message", message); - trace.put("details", traceDetails); - - try { - String loggableJsonString = mapper.writeValueAsString(trace); - tracer.info(loggableJsonString); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - public void initialize(PulsarService pulsarService) { - // defaultEnabledEvents.add(EventReasons.ADMINISTRATIVE); - // defaultEnabledEvents.add(EventReasons.MESSAGE); - // defaultEnabledEvents.add(EventReasons.TRANSACTION); + log.info("Initializing BrokerTracing"); } @Override - public void close() {} + public void close() { + log.info("Closing BrokerTracing"); + } private Set getEnabledEvents(ServerCnx cnx) { return getEnabledEvents(cnx.getBrokerService().getPulsar()); } private Set getEnabledEvents(PulsarService pulsar) { - // todo: do this less frequently - - String events = - pulsar.getConfiguration().getProperties().getProperty("jmsTracingEventList", ""); - - Set enabledEvents = new HashSet<>(); - - for (String event : events.split(",")) { - try { - enabledEvents.add(EventReasons.valueOf(event.trim())); - } catch (IllegalArgumentException e) { - log.warn("Invalid event: {}. Skipping", event); - } + try { + return jmsTracingEventList.get(pulsar); + } catch (ExecutionException e) { + log.error("Error getting enabled events", e); + return new HashSet<>(); } - - return enabledEvents; } private static TraceLevel getTracingLevel(ServerCnx cnx) { - // todo: do this less frequently - String level = - cnx.getBrokerService() - .getPulsar() - .getConfiguration() - .getProperties() - .getProperty("jmsTracingLevel", defaultTraceLevel.toString()); + try { - return TraceLevel.valueOf(level); - } catch (IllegalArgumentException e) { - log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel); + return traceLevelForService.get(cnx.getBrokerService().getPulsar()); + } catch (ExecutionException e) { + log.error("Error getting tracing level", e); return defaultTraceLevel; } } - public static Map getConnectionDetails(TraceLevel level, ServerCnx cnx) { - if (cnx == null) { - return null; - } - - Map details = new TreeMap<>(); - populateConnectionDetails(level, cnx, details); - return details; - } - - private static void populateConnectionDetails( - TraceLevel level, ServerCnx cnx, Map traceDetails) { - if (cnx == null) { - return; - } - - switch (level) { - case MINIMAL: - traceDetails.put("clientAddress", cnx.clientAddress().toString()); - break; - case BASIC: - populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); - - traceDetails.put("clientVersion", cnx.getClientVersion()); - traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); - break; - case FULL: - populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); - populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); - - traceDetails.put("authRole", cnx.getAuthRole()); - traceDetails.put("authMethod", cnx.getAuthMethod()); - traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } - } - - public static Map getSubscriptionDetails(TraceLevel level, Subscription sub) { - if (sub == null) { - return null; - } - - Map details = new TreeMap<>(); - populateSubscriptionDetails(level, sub, details); - return details; - } - - private static void populateSubscriptionDetails( - TraceLevel level, Subscription sub, Map traceDetails) { - if (sub == null) { - return; - } - - switch (level) { - case MINIMAL: - traceDetails.put("name", sub.getName()); - traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); - traceDetails.put("type", sub.getType().name()); - break; - case BASIC: - populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); - - if (sub.getConsumers() != null) { - traceDetails.put("numberOfConsumers", sub.getConsumers().size()); - traceDetails.put( - "namesOfConsumers", - sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList())); - } - - break; - case FULL: - populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); - populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails); - - traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } - } - - public static Map getConsumerDetails(TraceLevel level, Consumer consumer) { - if (consumer == null) { - return null; - } - - Map details = new TreeMap<>(); - populateConsumerDetails(level, consumer, details); - return details; - } - - private static void populateConsumerDetails( - TraceLevel level, Consumer consumer, Map traceDetails) { - if (consumer == null) { - return; - } - - switch (level) { - case MINIMAL: - traceDetails.put("name", consumer.consumerName()); - traceDetails.put("consumerId", consumer.consumerId()); - if (consumer.getSubscription() != null) { - traceDetails.put("subscriptionName", consumer.getSubscription().getName()); - traceDetails.put( - "topicName", - TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName()); - } - break; - case BASIC: - populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); - - traceDetails.put("priorityLevel", consumer.getPriorityLevel()); - traceDetails.put("subType", consumer.subType().name()); - traceDetails.put("clientAddress", consumer.getClientAddress()); - break; - case FULL: - populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); - populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails); - - traceDetails.put("metadata", consumer.getMetadata()); - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } - } - - public static Map getProducerDetails(TraceLevel level, Producer producer) { - if (producer == null) { - return null; - } - - Map details = new TreeMap<>(); - populateProducerDetails(level, producer, details); - return details; - } - - private static void populateProducerDetails( - TraceLevel level, Producer producer, Map traceDetails) { - if (producer == null) { - return; - } - - switch (level) { - case MINIMAL: - traceDetails.put("producerId", producer.getProducerId()); - traceDetails.put("producerName", producer.getProducerName()); - traceDetails.put("accessMode", producer.getAccessMode().name()); - if (producer.getTopic() != null) { - traceDetails.put( - "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); - } - break; - case BASIC: - populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); - - traceDetails.put("clientAddress", producer.getClientAddress()); - break; - case FULL: - populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); - populateProducerDetails(TraceLevel.BASIC, producer, traceDetails); - - traceDetails.put("metadata", producer.getMetadata()); - if (producer.getSchemaVersion() != null) { - traceDetails.put("schemaVersion", producer.getSchemaVersion().toString()); - } - traceDetails.put("remoteCluster", producer.getRemoteCluster()); - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } - } - - public static Map getMessageMetadataDetails( - TraceLevel level, MessageMetadata msgMetadata) { - if (msgMetadata == null) { - return null; - } - - Map details = new TreeMap<>(); - populateMessageMetadataDetails(level, msgMetadata, details); - return details; - } - - private static void populateMessageMetadataDetails( - TraceLevel level, MessageMetadata msgMetadata, Map traceDetails) { - if (msgMetadata == null) { - return; - } - - switch (level) { - case MINIMAL: - traceDetails.put("sequenceId", msgMetadata.getSequenceId()); - traceDetails.put("producerName", msgMetadata.getProducerName()); - traceDetails.put("partitionKey", msgMetadata.getPartitionKey()); - break; - case BASIC: - populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); - - traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); - traceDetails.put("serializedSize", msgMetadata.getSerializedSize()); - traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); - break; - case FULL: - populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); - populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails); - - traceDetails.put("publishTime", msgMetadata.getPublishTime()); - traceDetails.put("eventTime", msgMetadata.getEventTime()); - traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom()); - traceDetails.put("uuid", msgMetadata.getUuid()); - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } - } - - public static Map getEntryDetails(TraceLevel level, Entry entry) { - if (entry == null) { - return null; - } - - Map details = new TreeMap<>(); - populateEntryDetails(level, entry, details); - return details; - } - - private static void populateEntryDetails( - TraceLevel level, Entry entry, Map traceDetails) { - if (entry == null) { - return; - } - - switch (level) { - case MINIMAL: - traceDetails.put("ledgerId", entry.getLedgerId()); - traceDetails.put("entryId", entry.getEntryId()); - break; - case BASIC: - populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); - - traceDetails.put("length", entry.getLength()); - break; - case FULL: - populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); - populateEntryDetails(TraceLevel.BASIC, entry, traceDetails); - - traceByteBuf("data", entry.getDataBuffer(), traceDetails); - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } - } - - public static Map getPublishContextDetails(Topic.PublishContext publishContext) { - if (publishContext == null) { - return null; - } - - Map details = new TreeMap<>(); - populatePublishContext(publishContext, details); - return details; - } - - private static void populatePublishContext( - Topic.PublishContext publishContext, Map traceDetails) { - traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage()); - traceDetails.put("isChunked", publishContext.isChunked()); - traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages()); - - traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp()); - traceDetails.put("msgSize", publishContext.getMsgSize()); - traceDetails.put("producerName", publishContext.getProducerName()); - traceDetails.put("originalProducerName", publishContext.getOriginalProducerName()); - traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId()); - traceDetails.put("sequenceId", publishContext.getSequenceId()); - } - - private static void traceByteBuf(String key, ByteBuf buf, Map traceDetails) { - if (buf == null) return; - - if (buf.readableBytes() < MAX_DATA_LENGTH) { - traceDetails.put(key, Hex.encodeHex(buf.nioBuffer())); - } else { - traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); - } - } - private static TraceLevel getTracingLevel(Subscription sub) { if (sub == null || sub.getSubscriptionProperties() == null diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java new file mode 100644 index 00000000..c875e27e --- /dev/null +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -0,0 +1,361 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.pulsar.jms.tracing; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import io.netty.buffer.ByteBuf; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.commons.codec.binary.Hex; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Producer; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.naming.TopicName; + +@Slf4j +public class TracingUtils { + private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing"); + + private static final ObjectMapper mapper = + new ObjectMapper() + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) + .enable(SerializationFeature.WRITE_NULL_MAP_VALUES); + + public static final int MAX_DATA_LENGTH = 1024; + + public enum TraceLevel { + NONE, + MINIMAL, + BASIC, + FULL + } + + public static void trace(String message, Map traceDetails) { + Map trace = new TreeMap<>(); + trace.put("message", message); + trace.put("details", traceDetails); + + try { + String loggableJsonString = mapper.writeValueAsString(trace); + tracer.info(loggableJsonString); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static Map getConnectionDetails(TraceLevel level, ServerCnx cnx) { + if (cnx == null) { + return null; + } + + Map details = new TreeMap<>(); + populateConnectionDetails(level, cnx, details); + return details; + } + + private static void populateConnectionDetails( + TraceLevel level, ServerCnx cnx, Map traceDetails) { + if (cnx == null) { + return; + } + + switch (level) { + case MINIMAL: + traceDetails.put("clientAddress", cnx.clientAddress().toString()); + break; + case BASIC: + populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); + + traceDetails.put("clientVersion", cnx.getClientVersion()); + traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); + break; + case FULL: + populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); + populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); + + traceDetails.put("authRole", cnx.getAuthRole()); + traceDetails.put("authMethod", cnx.getAuthMethod()); + traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); + break; + default: + log.warn("Unknown tracing level: {}", level); + break; + } + } + + public static Map getSubscriptionDetails(TraceLevel level, Subscription sub) { + if (sub == null) { + return null; + } + + Map details = new TreeMap<>(); + populateSubscriptionDetails(level, sub, details); + return details; + } + + private static void populateSubscriptionDetails( + TraceLevel level, Subscription sub, Map traceDetails) { + if (sub == null) { + return; + } + + switch (level) { + case MINIMAL: + traceDetails.put("name", sub.getName()); + traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); + traceDetails.put("type", sub.getType().name()); + break; + case BASIC: + populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); + + if (sub.getConsumers() != null) { + traceDetails.put("numberOfConsumers", sub.getConsumers().size()); + traceDetails.put( + "namesOfConsumers", + sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList())); + } + + break; + case FULL: + populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); + populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails); + + traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); + break; + default: + log.warn("Unknown tracing level: {}", level); + break; + } + } + + public static Map getConsumerDetails(TraceLevel level, Consumer consumer) { + if (consumer == null) { + return null; + } + + Map details = new TreeMap<>(); + populateConsumerDetails(level, consumer, details); + return details; + } + + private static void populateConsumerDetails( + TraceLevel level, Consumer consumer, Map traceDetails) { + if (consumer == null) { + return; + } + + switch (level) { + case MINIMAL: + traceDetails.put("name", consumer.consumerName()); + traceDetails.put("consumerId", consumer.consumerId()); + if (consumer.getSubscription() != null) { + traceDetails.put("subscriptionName", consumer.getSubscription().getName()); + traceDetails.put( + "topicName", + TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName()); + } + break; + case BASIC: + populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); + + traceDetails.put("priorityLevel", consumer.getPriorityLevel()); + traceDetails.put("subType", consumer.subType().name()); + traceDetails.put("clientAddress", consumer.getClientAddress()); + break; + case FULL: + populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); + populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails); + + traceDetails.put("metadata", consumer.getMetadata()); + break; + default: + log.warn("Unknown tracing level: {}", level); + break; + } + } + + public static Map getProducerDetails(TraceLevel level, Producer producer) { + if (producer == null) { + return null; + } + + Map details = new TreeMap<>(); + populateProducerDetails(level, producer, details); + return details; + } + + private static void populateProducerDetails( + TraceLevel level, Producer producer, Map traceDetails) { + if (producer == null) { + return; + } + + switch (level) { + case MINIMAL: + traceDetails.put("producerId", producer.getProducerId()); + traceDetails.put("producerName", producer.getProducerName()); + traceDetails.put("accessMode", producer.getAccessMode().name()); + if (producer.getTopic() != null) { + traceDetails.put( + "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); + } + break; + case BASIC: + populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); + + traceDetails.put("clientAddress", producer.getClientAddress()); + break; + case FULL: + populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); + populateProducerDetails(TraceLevel.BASIC, producer, traceDetails); + + traceDetails.put("metadata", producer.getMetadata()); + if (producer.getSchemaVersion() != null) { + traceDetails.put("schemaVersion", producer.getSchemaVersion().toString()); + } + traceDetails.put("remoteCluster", producer.getRemoteCluster()); + break; + default: + log.warn("Unknown tracing level: {}", level); + break; + } + } + + public static Map getMessageMetadataDetails( + TraceLevel level, MessageMetadata msgMetadata) { + if (msgMetadata == null) { + return null; + } + + Map details = new TreeMap<>(); + populateMessageMetadataDetails(level, msgMetadata, details); + return details; + } + + private static void populateMessageMetadataDetails( + TraceLevel level, MessageMetadata msgMetadata, Map traceDetails) { + if (msgMetadata == null) { + return; + } + + switch (level) { + case MINIMAL: + traceDetails.put("sequenceId", msgMetadata.getSequenceId()); + traceDetails.put("producerName", msgMetadata.getProducerName()); + traceDetails.put("partitionKey", msgMetadata.getPartitionKey()); + break; + case BASIC: + populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); + + traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); + traceDetails.put("serializedSize", msgMetadata.getSerializedSize()); + traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); + break; + case FULL: + populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); + populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails); + + traceDetails.put("publishTime", msgMetadata.getPublishTime()); + traceDetails.put("eventTime", msgMetadata.getEventTime()); + traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom()); + traceDetails.put("uuid", msgMetadata.getUuid()); + break; + default: + log.warn("Unknown tracing level: {}", level); + break; + } + } + + public static Map getEntryDetails(TraceLevel level, Entry entry) { + if (entry == null) { + return null; + } + + Map details = new TreeMap<>(); + populateEntryDetails(level, entry, details); + return details; + } + + private static void populateEntryDetails( + TraceLevel level, Entry entry, Map traceDetails) { + if (entry == null) { + return; + } + + switch (level) { + case MINIMAL: + traceDetails.put("ledgerId", entry.getLedgerId()); + traceDetails.put("entryId", entry.getEntryId()); + break; + case BASIC: + populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); + + traceDetails.put("length", entry.getLength()); + break; + case FULL: + populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); + populateEntryDetails(TraceLevel.BASIC, entry, traceDetails); + + traceByteBuf("data", entry.getDataBuffer(), traceDetails); + break; + default: + log.warn("Unknown tracing level: {}", level); + break; + } + } + + public static Map getPublishContextDetails(Topic.PublishContext publishContext) { + if (publishContext == null) { + return null; + } + + Map details = new TreeMap<>(); + populatePublishContext(publishContext, details); + return details; + } + + private static void populatePublishContext( + Topic.PublishContext publishContext, Map traceDetails) { + traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage()); + traceDetails.put("isChunked", publishContext.isChunked()); + traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages()); + + traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp()); + traceDetails.put("msgSize", publishContext.getMsgSize()); + traceDetails.put("producerName", publishContext.getProducerName()); + traceDetails.put("originalProducerName", publishContext.getOriginalProducerName()); + traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId()); + traceDetails.put("sequenceId", publishContext.getSequenceId()); + } + + public static void traceByteBuf(String key, ByteBuf buf, Map traceDetails) { + if (buf == null) return; + + if (buf.readableBytes() < MAX_DATA_LENGTH) { + traceDetails.put(key, Hex.encodeHex(buf.nioBuffer())); + } else { + traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); + } + } +} From 2246bc44efc58596f8976f255f85e8cdf40cb8eb Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Fri, 12 Apr 2024 16:42:36 -0700 Subject: [PATCH 05/29] more config caching --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 101 ++++++++++++------ 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index d3b7f248..3e0b91b8 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.servlet.ServletException; import javax.servlet.ServletRequest; @@ -57,10 +58,19 @@ @Slf4j public class BrokerTracing implements BrokerInterceptor { + public enum EventReasons { + ADMINISTRATIVE, + MESSAGE, + TRANSACTION, + SERVLET, + } + + private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC; + private static final LoadingCache> jmsTracingEventList = CacheBuilder.newBuilder() .maximumSize(10) - .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES) + .expireAfterWrite(1, TimeUnit.MINUTES) .build( new CacheLoader>() { public Set load(PulsarService pulsar) { @@ -89,7 +99,7 @@ public Set load(PulsarService pulsar) { private static final LoadingCache traceLevelForService = CacheBuilder.newBuilder() .maximumSize(10) - .expireAfterWrite(1, java.util.concurrent.TimeUnit.MINUTES) + .expireAfterWrite(1, TimeUnit.MINUTES) .build( new CacheLoader() { public TraceLevel load(PulsarService pulsar) { @@ -108,14 +118,50 @@ public TraceLevel load(PulsarService pulsar) { } }); - public enum EventReasons { - ADMINISTRATIVE, - MESSAGE, - TRANSACTION, - SERVLET, - } + private static final LoadingCache traceLevelForSubscription = + CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.SECONDS) + .build( + new CacheLoader() { + public TraceLevel load(Subscription sub) { + if (sub.getSubscriptionProperties() == null + || !sub.getSubscriptionProperties().containsKey("trace")) { + return TraceLevel.NONE; + } - private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC; + try { + return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace")); + } catch (IllegalArgumentException e) { + log.warn( + "Invalid tracing level: {}. Setting to NONE for subscription {}", + sub.getSubscriptionProperties().get("trace"), + sub); + return TraceLevel.NONE; + } + } + }); + + private static final LoadingCache traceLevelForProducer = + CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.SECONDS) + .build( + new CacheLoader() { + public TraceLevel load(Producer producer) { + if (producer.getMetadata() == null + || !producer.getMetadata().containsKey("trace")) { + return TraceLevel.NONE; + } + try { + return TraceLevel.valueOf(producer.getMetadata().get("trace")); + } catch (IllegalArgumentException e) { + log.warn( + "Invalid tracing level: {}. Setting to NONE for producer {}", + producer.getMetadata().get("trace"), + producer); + return TraceLevel.NONE; + } + } + }); public void initialize(PulsarService pulsarService) { log.info("Initializing BrokerTracing"); @@ -127,10 +173,14 @@ public void close() { } private Set getEnabledEvents(ServerCnx cnx) { + if (cnx == null) return new HashSet<>(); + return getEnabledEvents(cnx.getBrokerService().getPulsar()); } private Set getEnabledEvents(PulsarService pulsar) { + if (pulsar == null) return new HashSet<>(); + try { return jmsTracingEventList.get(pulsar); } catch (ExecutionException e) { @@ -140,6 +190,7 @@ private Set getEnabledEvents(PulsarService pulsar) { } private static TraceLevel getTracingLevel(ServerCnx cnx) { + if (cnx == null) return defaultTraceLevel; try { return traceLevelForService.get(cnx.getBrokerService().getPulsar()); @@ -150,35 +201,23 @@ private static TraceLevel getTracingLevel(ServerCnx cnx) { } private static TraceLevel getTracingLevel(Subscription sub) { - if (sub == null - || sub.getSubscriptionProperties() == null - || !sub.getSubscriptionProperties().containsKey("trace")) { - return TraceLevel.NONE; - } + if (sub == null) return TraceLevel.NONE; + try { - return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace")); - } catch (IllegalArgumentException e) { - log.warn( - "Invalid tracing level: {}. Setting to NONE for subscription {}", - sub.getSubscriptionProperties().get("trace"), - sub); + return traceLevelForSubscription.get(sub); + } catch (ExecutionException e) { + log.error("Error getting tracing level", e); return TraceLevel.NONE; } } private static TraceLevel getTracingLevel(Producer producer) { - if (producer == null - || producer.getMetadata() == null - || !producer.getMetadata().containsKey("trace")) { - return TraceLevel.NONE; - } + if (producer == null) return TraceLevel.NONE; + try { - return TraceLevel.valueOf(producer.getMetadata().get("trace")); - } catch (IllegalArgumentException e) { - log.warn( - "Invalid tracing level: {}. Setting to NONE for producer {}", - producer.getMetadata().get("trace"), - producer); + return traceLevelForProducer.get(producer); + } catch (ExecutionException e) { + log.error("Error getting tracing level", e); return TraceLevel.NONE; } } From 4f84dc7dc795e09bfc8690caa8412b328dd47009 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Fri, 12 Apr 2024 17:21:29 -0700 Subject: [PATCH 06/29] couple of tests --- .../oss/pulsar/jms/tracing/TracingUtils.java | 22 +++- .../pulsar/jms/tracing/TracingUtilsTest.java | 103 ++++++++++++++++++ 2 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index c875e27e..fbe4d88b 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -35,7 +35,14 @@ @Slf4j public class TracingUtils { - private static final org.slf4j.Logger tracer = org.slf4j.LoggerFactory.getLogger("jms-tracing"); + private static final org.slf4j.Logger traceLogger = org.slf4j.LoggerFactory.getLogger("jms-tracing"); + + @FunctionalInterface + public interface Tracer { + void trace(String message); + } + + public static final Tracer SLF4J_TRACER = traceLogger::info; private static final ObjectMapper mapper = new ObjectMapper() @@ -51,14 +58,19 @@ public enum TraceLevel { FULL } + public static void trace(String message, Map traceDetails) { + trace(SLF4J_TRACER, message, traceDetails); + } + + public static void trace(Tracer tracer, String message, Map traceDetails) { Map trace = new TreeMap<>(); trace.put("message", message); - trace.put("details", traceDetails); + trace.put("traceDetails", traceDetails); try { String loggableJsonString = mapper.writeValueAsString(trace); - tracer.info(loggableJsonString); + tracer.trace(loggableJsonString); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -353,9 +365,9 @@ public static void traceByteBuf(String key, ByteBuf buf, Map tra if (buf == null) return; if (buf.readableBytes() < MAX_DATA_LENGTH) { - traceDetails.put(key, Hex.encodeHex(buf.nioBuffer())); + traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer())); } else { - traceDetails.put(key + "Slice", Hex.encodeHex(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); + traceDetails.put(key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); } } } diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java new file mode 100644 index 00000000..956034cf --- /dev/null +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -0,0 +1,103 @@ +package com.datastax.oss.pulsar.jms.tracing; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.jupiter.api.Assertions.*; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*; + +class TracingUtilsTest { + + List traces = new ArrayList<>(); + private Tracer mockTracer = new Tracer() { + @Override + public void trace(String msg) { + traces.add(msg); + } + }; + + @Test + void traceTest() { + traces.clear(); + trace(mockTracer, "msg", null); + assertEquals(1, traces.size()); + assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0)); + + Map map = new TreeMap<>(); + + traces.clear(); + trace(mockTracer, "msg", map); + assertEquals(1, traces.size()); + assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0)); + + map.put("key1", "value1"); + + traces.clear(); + trace(mockTracer, "msg", map); + assertEquals(1, traces.size()); + assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); + } + +// todo: +// @Test +// void getConnectionDetailsTest() { +// } +// +// @Test +// void getSubscriptionDetailsTest() { +// } +// +// @Test +// void getConsumerDetailsTest() { +// } +// +// @Test +// void getProducerDetails() { +// } +// +// @Test +// void getMessageMetadataDetailsTest() { +// } +// +// @Test +// void getEntryDetailsTest() { +// } +// +// @Test +// void getPublishContextDetailsTest() { +// } + + @Test + void traceByteBufTest() { + Map traceDetails = new TreeMap<>(); + + traceByteBuf("key", null, traceDetails); + assertEquals(0, traceDetails.size()); + + ByteBuf small = Unpooled.buffer(20); + for (int i = 0; i < 20; i++) { + small.writeByte(i); + } + traceByteBuf("key", small, traceDetails); + assertEquals(1, traceDetails.size()); + assertEquals(42, ((String) traceDetails.get("key")).length()); + assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key")); + + ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100); + for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) { + big.writeByte(i); + } + + traceDetails.clear(); + traceByteBuf("key", big, traceDetails); + assertEquals(1, traceDetails.size()); + assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length()); + assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102")); + } +} \ No newline at end of file From ffc604eeab31d91ecbca6563e026df8714669247 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 15 Apr 2024 10:24:24 -0700 Subject: [PATCH 07/29] feedback --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 184 +++++++---------- .../oss/pulsar/jms/tracing/TracingUtils.java | 13 +- .../pulsar/jms/tracing/TracingUtilsTest.java | 189 ++++++++++-------- 3 files changed, 184 insertions(+), 202 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 3e0b91b8..a1d7062b 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -54,6 +54,7 @@ import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.intercept.InterceptException; +import org.jetbrains.annotations.NotNull; @Slf4j public class BrokerTracing implements BrokerInterceptor { @@ -67,56 +68,40 @@ public enum EventReasons { private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC; - private static final LoadingCache> jmsTracingEventList = - CacheBuilder.newBuilder() - .maximumSize(10) - .expireAfterWrite(1, TimeUnit.MINUTES) - .build( - new CacheLoader>() { - public Set load(PulsarService pulsar) { - // dynamic config can get reloaded without service restart - String events = - pulsar - .getConfiguration() - .getProperties() - .getProperty("jmsTracingEventList", ""); - log.debug("read jmsTracingEventList: {}", events); - - Set enabledEvents = new HashSet<>(); - - for (String event : events.split(",")) { - try { - enabledEvents.add(EventReasons.valueOf(event.trim())); - } catch (IllegalArgumentException e) { - log.warn("Invalid event: {}. Skipping", event); - } - } + private final Set jmsTracingEventList = new HashSet<>(); + private TraceLevel traceLevel = defaultTraceLevel; + + private static Set loadEnabledEvents( + PulsarService pulsarService, Set enabledEvents) { + String events = + pulsarService.getConfiguration().getProperties().getProperty("jmsTracingEventList", ""); + log.debug("read jmsTracingEventList: {}", events); + + for (String event : events.split(",")) { + try { + enabledEvents.add(EventReasons.valueOf(event.trim())); + } catch (IllegalArgumentException e) { + log.error("Invalid event: {}. Skipping", event); + } + } - return enabledEvents; - } - }); + return enabledEvents; + } - private static final LoadingCache traceLevelForService = - CacheBuilder.newBuilder() - .maximumSize(10) - .expireAfterWrite(1, TimeUnit.MINUTES) - .build( - new CacheLoader() { - public TraceLevel load(PulsarService pulsar) { - String level = - pulsar - .getConfiguration() - .getProperties() - .getProperty("jmsTracingLevel", defaultTraceLevel.toString()); - try { - return TraceLevel.valueOf(level); - } catch (IllegalArgumentException e) { - log.warn( - "Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel); - return defaultTraceLevel; - } - } - }); + @NotNull + private static TraceLevel getTraceLevel(PulsarService pulsar) { + String level = + pulsar + .getConfiguration() + .getProperties() + .getProperty("jmsTracingLevel", defaultTraceLevel.toString()); + try { + return TraceLevel.valueOf(level); + } catch (IllegalArgumentException e) { + log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel); + return defaultTraceLevel; + } + } private static final LoadingCache traceLevelForSubscription = CacheBuilder.newBuilder() @@ -124,17 +109,17 @@ public TraceLevel load(PulsarService pulsar) { .build( new CacheLoader() { public TraceLevel load(Subscription sub) { - if (sub.getSubscriptionProperties() == null - || !sub.getSubscriptionProperties().containsKey("trace")) { + Map subProps = sub.getSubscriptionProperties(); + if (subProps == null || !subProps.containsKey("trace")) { return TraceLevel.NONE; } try { - return TraceLevel.valueOf(sub.getSubscriptionProperties().get("trace")); + return TraceLevel.valueOf(subProps.get("trace")); } catch (IllegalArgumentException e) { log.warn( "Invalid tracing level: {}. Setting to NONE for subscription {}", - sub.getSubscriptionProperties().get("trace"), + subProps.get("trace"), sub); return TraceLevel.NONE; } @@ -165,6 +150,9 @@ public TraceLevel load(Producer producer) { public void initialize(PulsarService pulsarService) { log.info("Initializing BrokerTracing"); + + loadEnabledEvents(pulsarService, jmsTracingEventList); + traceLevel = getTraceLevel(pulsarService); } @Override @@ -172,32 +160,9 @@ public void close() { log.info("Closing BrokerTracing"); } - private Set getEnabledEvents(ServerCnx cnx) { - if (cnx == null) return new HashSet<>(); - - return getEnabledEvents(cnx.getBrokerService().getPulsar()); - } - - private Set getEnabledEvents(PulsarService pulsar) { - if (pulsar == null) return new HashSet<>(); - - try { - return jmsTracingEventList.get(pulsar); - } catch (ExecutionException e) { - log.error("Error getting enabled events", e); - return new HashSet<>(); - } - } - - private static TraceLevel getTracingLevel(ServerCnx cnx) { - if (cnx == null) return defaultTraceLevel; - - try { - return traceLevelForService.get(cnx.getBrokerService().getPulsar()); - } catch (ExecutionException e) { - log.error("Error getting tracing level", e); - return defaultTraceLevel; - } + private static TraceLevel getTracingLevel(Consumer consumer) { + if (consumer == null) return TraceLevel.NONE; + return getTracingLevel(consumer.getSubscription()); } private static TraceLevel getTracingLevel(Subscription sub) { @@ -234,20 +199,19 @@ private static TraceLevel getTracingLevel(Producer producer) { ******************************/ public void onConnectionCreated(ServerCnx cnx) { - if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - TraceLevel level = getTracingLevel(cnx); - if (level == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); trace("Connection created", traceDetails); } public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { - if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - TraceLevel level = getTracingLevel(cnx); + TraceLevel level = getTracingLevel(producer); if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); @@ -259,9 +223,9 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { - if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - TraceLevel level = getTracingLevel(cnx); + TraceLevel level = getTracingLevel(producer); if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); @@ -273,9 +237,9 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map } public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { - if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - TraceLevel level = getTracingLevel(cnx); + TraceLevel level = getTracingLevel(consumer); if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); @@ -288,9 +252,9 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { - if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - TraceLevel level = getTracingLevel(cnx); + TraceLevel level = getTracingLevel(consumer); if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); @@ -303,26 +267,24 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map } public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { - if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - TraceLevel level = getTracingLevel(cnx); - if (level == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); traceDetails.put("command", command.toString()); trace("Pulsar command called", traceDetails); } public void onConnectionClosed(ServerCnx cnx) { - if (!getEnabledEvents(cnx).contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - TraceLevel level = getTracingLevel(cnx); - if (level == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); trace("Connection closed", traceDetails); } @@ -337,8 +299,7 @@ public void beforeSendMessage( long[] ackSet, MessageMetadata msgMetadata, Consumer consumer) { - if (!getEnabledEvents(consumer.cnx().getBrokerService().getPulsar()) - .contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(subscription); if (level == TraceLevel.NONE) return; @@ -355,8 +316,7 @@ public void beforeSendMessage( public void onMessagePublish( Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) { - if (!getEnabledEvents(producer.getCnx().getBrokerService().getPulsar()) - .contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(producer); if (level == TraceLevel.NONE) return; @@ -376,7 +336,7 @@ public void messageProduced( long ledgerId, long entryId, Topic.PublishContext publishContext) { - if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(producer); if (level == TraceLevel.NONE) return; @@ -394,9 +354,9 @@ public void messageProduced( public void messageDispatched( ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) { - if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; - TraceLevel level = getTracingLevel(consumer.getSubscription()); + TraceLevel level = getTracingLevel(consumer); if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); @@ -411,9 +371,9 @@ public void messageDispatched( } public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { - if (!getEnabledEvents(cnx).contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; - TraceLevel level = getTracingLevel(consumer.getSubscription()); + TraceLevel level = getTracingLevel(consumer); if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); @@ -442,23 +402,25 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { ******************************/ public void txnOpened(long tcId, String txnID) { - // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) { + if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return; + if (traceLevel == TraceLevel.NONE) return; + Map traceDetails = new TreeMap<>(); traceDetails.put("tcId", tcId); traceDetails.put("txnID", txnID); trace("Transaction opened", traceDetails); - // } } public void txnEnded(String txnID, long txnAction) { - // if (getEnabledEvents(???).contains(EventReasons.TRANSACTION)) { + if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return; + if (traceLevel == TraceLevel.NONE) return; + Map traceDetails = new TreeMap<>(); traceDetails.put("txnID", txnID); traceDetails.put("txnAction", txnAction); trace("Transaction closed", traceDetails); - // } } /* *************************** diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index fbe4d88b..bed9bf17 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -35,7 +35,8 @@ @Slf4j public class TracingUtils { - private static final org.slf4j.Logger traceLogger = org.slf4j.LoggerFactory.getLogger("jms-tracing"); + private static final org.slf4j.Logger traceLogger = + org.slf4j.LoggerFactory.getLogger("jms-tracing"); @FunctionalInterface public interface Tracer { @@ -58,7 +59,6 @@ public enum TraceLevel { FULL } - public static void trace(String message, Map traceDetails) { trace(SLF4J_TRACER, message, traceDetails); } @@ -72,7 +72,11 @@ public static void trace(Tracer tracer, String message, Map trac String loggableJsonString = mapper.writeValueAsString(trace); tracer.trace(loggableJsonString); } catch (JsonProcessingException e) { - throw new RuntimeException(e); + log.error( + "Failed to serialize trace message '{}' as json, traceDetails: {}", + message, + traceDetails, + e); } } @@ -367,7 +371,8 @@ public static void traceByteBuf(String key, ByteBuf buf, Map tra if (buf.readableBytes() < MAX_DATA_LENGTH) { traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer())); } else { - traceDetails.put(key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); + traceDetails.put( + key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); } } } diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java index 956034cf..4675de18 100644 --- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -1,103 +1,118 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.datastax.oss.pulsar.jms.tracing; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*; +import static org.junit.jupiter.api.Assertions.*; + import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; -import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; - -import static org.junit.jupiter.api.Assertions.*; -import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.*; +import org.junit.jupiter.api.Test; class TracingUtilsTest { - List traces = new ArrayList<>(); - private Tracer mockTracer = new Tracer() { + List traces = new ArrayList<>(); + private Tracer mockTracer = + new Tracer() { @Override public void trace(String msg) { - traces.add(msg); + traces.add(msg); } - }; - - @Test - void traceTest() { - traces.clear(); - trace(mockTracer, "msg", null); - assertEquals(1, traces.size()); - assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0)); - - Map map = new TreeMap<>(); - - traces.clear(); - trace(mockTracer, "msg", map); - assertEquals(1, traces.size()); - assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0)); - - map.put("key1", "value1"); - - traces.clear(); - trace(mockTracer, "msg", map); - assertEquals(1, traces.size()); - assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); + }; + + @Test + void traceTest() { + traces.clear(); + trace(mockTracer, "msg", null); + assertEquals(1, traces.size()); + assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0)); + + Map map = new TreeMap<>(); + + traces.clear(); + trace(mockTracer, "msg", map); + assertEquals(1, traces.size()); + assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0)); + + map.put("key1", "value1"); + + traces.clear(); + trace(mockTracer, "msg", map); + assertEquals(1, traces.size()); + assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); + } + + // todo: + // @Test + // void getConnectionDetailsTest() { + // } + // + // @Test + // void getSubscriptionDetailsTest() { + // } + // + // @Test + // void getConsumerDetailsTest() { + // } + // + // @Test + // void getProducerDetails() { + // } + // + // @Test + // void getMessageMetadataDetailsTest() { + // } + // + // @Test + // void getEntryDetailsTest() { + // } + // + // @Test + // void getPublishContextDetailsTest() { + // } + + @Test + void traceByteBufTest() { + Map traceDetails = new TreeMap<>(); + + traceByteBuf("key", null, traceDetails); + assertEquals(0, traceDetails.size()); + + ByteBuf small = Unpooled.buffer(20); + for (int i = 0; i < 20; i++) { + small.writeByte(i); } - -// todo: -// @Test -// void getConnectionDetailsTest() { -// } -// -// @Test -// void getSubscriptionDetailsTest() { -// } -// -// @Test -// void getConsumerDetailsTest() { -// } -// -// @Test -// void getProducerDetails() { -// } -// -// @Test -// void getMessageMetadataDetailsTest() { -// } -// -// @Test -// void getEntryDetailsTest() { -// } -// -// @Test -// void getPublishContextDetailsTest() { -// } - - @Test - void traceByteBufTest() { - Map traceDetails = new TreeMap<>(); - - traceByteBuf("key", null, traceDetails); - assertEquals(0, traceDetails.size()); - - ByteBuf small = Unpooled.buffer(20); - for (int i = 0; i < 20; i++) { - small.writeByte(i); - } - traceByteBuf("key", small, traceDetails); - assertEquals(1, traceDetails.size()); - assertEquals(42, ((String) traceDetails.get("key")).length()); - assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key")); - - ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100); - for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) { - big.writeByte(i); - } - - traceDetails.clear(); - traceByteBuf("key", big, traceDetails); - assertEquals(1, traceDetails.size()); - assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length()); - assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102")); + traceByteBuf("key", small, traceDetails); + assertEquals(1, traceDetails.size()); + assertEquals(42, ((String) traceDetails.get("key")).length()); + assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key")); + + ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100); + for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) { + big.writeByte(i); } -} \ No newline at end of file + + traceDetails.clear(); + traceByteBuf("key", big, traceDetails); + assertEquals(1, traceDetails.size()); + assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length()); + assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102")); + } +} From 4585bcab8266c7f3b337249dbc3e600bbaed6468 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 15 Apr 2024 14:19:26 -0700 Subject: [PATCH 08/29] fixes, tweaks --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 12 ++-- .../oss/pulsar/jms/tracing/TracingUtils.java | 62 ++++++++++++------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index a1d7062b..2b94a263 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -79,7 +79,7 @@ private static Set loadEnabledEvents( for (String event : events.split(",")) { try { - enabledEvents.add(EventReasons.valueOf(event.trim())); + enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase())); } catch (IllegalArgumentException e) { log.error("Invalid event: {}. Skipping", event); } @@ -96,7 +96,7 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) { .getProperties() .getProperty("jmsTracingLevel", defaultTraceLevel.toString()); try { - return TraceLevel.valueOf(level); + return TraceLevel.valueOf(level.trim().toUpperCase()); } catch (IllegalArgumentException e) { log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel); return defaultTraceLevel; @@ -115,7 +115,7 @@ public TraceLevel load(Subscription sub) { } try { - return TraceLevel.valueOf(subProps.get("trace")); + return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase()); } catch (IllegalArgumentException e) { log.warn( "Invalid tracing level: {}. Setting to NONE for subscription {}", @@ -137,7 +137,8 @@ public TraceLevel load(Producer producer) { return TraceLevel.NONE; } try { - return TraceLevel.valueOf(producer.getMetadata().get("trace")); + return TraceLevel.valueOf( + producer.getMetadata().get("trace").trim().toUpperCase()); } catch (IllegalArgumentException e) { log.warn( "Invalid tracing level: {}. Setting to NONE for producer {}", @@ -273,6 +274,9 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept Map traceDetails = new TreeMap<>(); traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); + // todo: .toString() is not good enough + // {"message":"Pulsar command + // called","traceDetails":{"command":"org.apache.pulsar.common.api.proto.BaseCommand@2822d70c","serverCnx":{"authMethod":"none","authMethodName":"no provider","authRole":null,"clientAddress":"/127.0.0.1:54176","clientSourceAddressAndPort":"/127.0.0.1:54176","clientVersion":"Pulsar-Java-v3.1.3.1-SNAPSHOT"}}} traceDetails.put("command", command.toString()); trace("Pulsar command called", traceDetails); diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index bed9bf17..372e6ca9 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -107,12 +107,15 @@ private static void populateConnectionDetails( traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); break; case FULL: - populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); traceDetails.put("authRole", cnx.getAuthRole()); traceDetails.put("authMethod", cnx.getAuthMethod()); - traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); + traceDetails.put( + "authMethodName", + cnx.getAuthenticationProvider() == null + ? "no provider" + : cnx.getAuthenticationProvider().getAuthMethodName()); break; default: log.warn("Unknown tracing level: {}", level); @@ -154,7 +157,6 @@ private static void populateSubscriptionDetails( break; case FULL: - populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails); traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); @@ -185,22 +187,21 @@ private static void populateConsumerDetails( case MINIMAL: traceDetails.put("name", consumer.consumerName()); traceDetails.put("consumerId", consumer.consumerId()); - if (consumer.getSubscription() != null) { - traceDetails.put("subscriptionName", consumer.getSubscription().getName()); + Subscription sub = consumer.getSubscription(); + if (sub != null) { + traceDetails.put("subscriptionName", sub.getName()); traceDetails.put( - "topicName", - TopicName.get(consumer.getSubscription().getTopicName()).getPartitionedTopicName()); + "topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); } break; case BASIC: populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); traceDetails.put("priorityLevel", consumer.getPriorityLevel()); - traceDetails.put("subType", consumer.subType().name()); + traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name()); traceDetails.put("clientAddress", consumer.getClientAddress()); break; case FULL: - populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails); traceDetails.put("metadata", consumer.getMetadata()); @@ -231,7 +232,9 @@ private static void populateProducerDetails( case MINIMAL: traceDetails.put("producerId", producer.getProducerId()); traceDetails.put("producerName", producer.getProducerName()); - traceDetails.put("accessMode", producer.getAccessMode().name()); + traceDetails.put( + "accessMode", + producer.getAccessMode() == null ? null : producer.getAccessMode().name()); if (producer.getTopic() != null) { traceDetails.put( "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); @@ -243,7 +246,6 @@ private static void populateProducerDetails( traceDetails.put("clientAddress", producer.getClientAddress()); break; case FULL: - populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); populateProducerDetails(TraceLevel.BASIC, producer, traceDetails); traceDetails.put("metadata", producer.getMetadata()); @@ -277,25 +279,42 @@ private static void populateMessageMetadataDetails( switch (level) { case MINIMAL: - traceDetails.put("sequenceId", msgMetadata.getSequenceId()); - traceDetails.put("producerName", msgMetadata.getProducerName()); - traceDetails.put("partitionKey", msgMetadata.getPartitionKey()); + if (msgMetadata.hasPartitionKey()) { + traceDetails.put("partitionKey", msgMetadata.getPartitionKey()); + } + if (msgMetadata.hasSequenceId()) { + traceDetails.put("sequenceId", msgMetadata.getSequenceId()); + } + if (msgMetadata.hasProducerName()) { + traceDetails.put("producerName", msgMetadata.getProducerName()); + } break; case BASIC: populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); - traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); + if (msgMetadata.hasUncompressedSize()) { + traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); + } + if (msgMetadata.hasNumMessagesInBatch()) { + traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); + } traceDetails.put("serializedSize", msgMetadata.getSerializedSize()); - traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); break; case FULL: - populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails); - traceDetails.put("publishTime", msgMetadata.getPublishTime()); - traceDetails.put("eventTime", msgMetadata.getEventTime()); - traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom()); - traceDetails.put("uuid", msgMetadata.getUuid()); + if (msgMetadata.hasPublishTime()) { + traceDetails.put("publishTime", msgMetadata.getPublishTime()); + } + if (msgMetadata.hasEventTime()) { + traceDetails.put("eventTime", msgMetadata.getEventTime()); + } + if (msgMetadata.hasReplicatedFrom()) { + traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom()); + } + if (msgMetadata.hasUuid()) { + traceDetails.put("uuid", msgMetadata.getUuid()); + } break; default: log.warn("Unknown tracing level: {}", level); @@ -330,7 +349,6 @@ private static void populateEntryDetails( traceDetails.put("length", entry.getLength()); break; case FULL: - populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); populateEntryDetails(TraceLevel.BASIC, entry, traceDetails); traceByteBuf("data", entry.getDataBuffer(), traceDetails); From 5e9094a10a027e2d25711acb8c958cc2c2d0b169 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 15 Apr 2024 16:20:37 -0700 Subject: [PATCH 09/29] actually trace commands; 'message' changed to 'eventType' --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 12 +- .../oss/pulsar/jms/tracing/TracingUtils.java | 301 +++++++++++++++++- 2 files changed, 307 insertions(+), 6 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 2b94a263..7606563f 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -16,6 +16,7 @@ package com.datastax.oss.pulsar.jms.tracing; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails; @@ -274,10 +275,13 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept Map traceDetails = new TreeMap<>(); traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); - // todo: .toString() is not good enough - // {"message":"Pulsar command - // called","traceDetails":{"command":"org.apache.pulsar.common.api.proto.BaseCommand@2822d70c","serverCnx":{"authMethod":"none","authMethodName":"no provider","authRole":null,"clientAddress":"/127.0.0.1:54176","clientSourceAddressAndPort":"/127.0.0.1:54176","clientVersion":"Pulsar-Java-v3.1.3.1-SNAPSHOT"}}} - traceDetails.put("command", command.toString()); + + if (command.hasType()) { + traceDetails.put("type", command.getType().name()); + traceDetails.put("command", getCommandDetails(traceLevel, command)); + } else { + traceDetails.put("type", "unknown/null"); + } trace("Pulsar command called", traceDetails); } diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 372e6ca9..cc6ba6de 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -19,7 +19,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import io.netty.buffer.ByteBuf; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -30,6 +34,7 @@ import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicName; @@ -65,7 +70,7 @@ public static void trace(String message, Map traceDetails) { public static void trace(Tracer tracer, String message, Map traceDetails) { Map trace = new TreeMap<>(); - trace.put("message", message); + trace.put("eventType", message); trace.put("traceDetails", traceDetails); try { @@ -73,13 +78,293 @@ public static void trace(Tracer tracer, String message, Map trac tracer.trace(loggableJsonString); } catch (JsonProcessingException e) { log.error( - "Failed to serialize trace message '{}' as json, traceDetails: {}", + "Failed to serialize trace event type '{}' as json, traceDetails: {}", message, traceDetails, e); } } + public static Map getCommandDetails(TraceLevel level, BaseCommand command) { + if (command == null) { + return null; + } + + Map details = new TreeMap<>(); + populateCommandDetails(level, command, details); + return details; + } + + private static void populateCommandDetails( + TraceLevel level, BaseCommand command, Map traceDetails) { + if (command == null) { + return; + } + if (level == TraceLevel.NONE) { + return; + } + + if (!command.hasType()) { + return; + } + + // trace all params otherwise + + switch (command.getType()) { + case CONNECT: + populateByReflection(command.getConnect(), traceDetails); + break; + case CONNECTED: + populateByReflection(command.getConnected(), traceDetails); + break; + case SUBSCRIBE: + populateByReflection(command.getSubscribe(), traceDetails); + break; + case PRODUCER: + populateByReflection(command.getProducer(), traceDetails); + break; + case SEND: + populateByReflection(command.getSend(), traceDetails); + break; + case SEND_RECEIPT: + populateByReflection(command.getSendReceipt(), traceDetails); + break; + case SEND_ERROR: + populateByReflection(command.getSendError(), traceDetails); + break; + case MESSAGE: + populateByReflection(command.getMessage(), traceDetails); + break; + case ACK: + populateByReflection(command.getAck(), traceDetails); + break; + case FLOW: + populateByReflection(command.getFlow(), traceDetails); + break; + case UNSUBSCRIBE: + populateByReflection(command.getUnsubscribe(), traceDetails); + break; + case SUCCESS: + populateByReflection(command.getSuccess(), traceDetails); + break; + case ERROR: + populateByReflection(command.getError(), traceDetails); + break; + case CLOSE_PRODUCER: + populateByReflection(command.getCloseProducer(), traceDetails); + break; + case CLOSE_CONSUMER: + populateByReflection(command.getCloseConsumer(), traceDetails); + break; + case PRODUCER_SUCCESS: + populateByReflection(command.getProducerSuccess(), traceDetails); + break; + case PING: + populateByReflection(command.getPing(), traceDetails); + break; + case PONG: + populateByReflection(command.getPong(), traceDetails); + break; + case REDELIVER_UNACKNOWLEDGED_MESSAGES: + populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails); + break; + case PARTITIONED_METADATA: + populateByReflection(command.getPartitionMetadata(), traceDetails); + break; + case PARTITIONED_METADATA_RESPONSE: + populateByReflection(command.getPartitionMetadataResponse(), traceDetails); + break; + case LOOKUP: + populateByReflection(command.getLookupTopic(), traceDetails); + break; + case LOOKUP_RESPONSE: + populateByReflection(command.getLookupTopicResponse(), traceDetails); + break; + case CONSUMER_STATS: + populateByReflection(command.getConsumerStats(), traceDetails); + break; + case CONSUMER_STATS_RESPONSE: + populateByReflection(command.getConsumerStatsResponse(), traceDetails); + break; + case REACHED_END_OF_TOPIC: + populateByReflection(command.getReachedEndOfTopic(), traceDetails); + break; + case SEEK: + populateByReflection(command.getSeek(), traceDetails); + break; + case GET_LAST_MESSAGE_ID: + populateByReflection(command.getGetLastMessageId(), traceDetails); + break; + case GET_LAST_MESSAGE_ID_RESPONSE: + populateByReflection(command.getGetLastMessageIdResponse(), traceDetails); + break; + case ACTIVE_CONSUMER_CHANGE: + populateByReflection(command.getActiveConsumerChange(), traceDetails); + break; + case GET_TOPICS_OF_NAMESPACE: + populateByReflection(command.getGetTopicsOfNamespace(), traceDetails); + break; + case GET_TOPICS_OF_NAMESPACE_RESPONSE: + populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails); + break; + case GET_SCHEMA: + populateByReflection(command.getGetSchema(), traceDetails); + break; + case GET_SCHEMA_RESPONSE: + populateByReflection(command.getGetSchemaResponse(), traceDetails); + break; + case AUTH_CHALLENGE: + populateByReflection(command.getAuthChallenge(), traceDetails); + break; + case AUTH_RESPONSE: + populateByReflection(command.getAuthResponse(), traceDetails); + break; + case ACK_RESPONSE: + populateByReflection(command.getAckResponse(), traceDetails); + break; + case GET_OR_CREATE_SCHEMA: + populateByReflection(command.getGetOrCreateSchema(), traceDetails); + break; + case GET_OR_CREATE_SCHEMA_RESPONSE: + populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails); + break; + case NEW_TXN: + populateByReflection(command.getNewTxn(), traceDetails); + break; + case NEW_TXN_RESPONSE: + populateByReflection(command.getNewTxnResponse(), traceDetails); + break; + case ADD_PARTITION_TO_TXN: + populateByReflection(command.getAddPartitionToTxn(), traceDetails); + break; + case ADD_PARTITION_TO_TXN_RESPONSE: + populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails); + break; + case ADD_SUBSCRIPTION_TO_TXN: + populateByReflection(command.getAddSubscriptionToTxn(), traceDetails); + break; + case ADD_SUBSCRIPTION_TO_TXN_RESPONSE: + populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails); + break; + case END_TXN: + populateByReflection(command.getEndTxn(), traceDetails); + break; + case END_TXN_RESPONSE: + populateByReflection(command.getEndTxnResponse(), traceDetails); + break; + case END_TXN_ON_PARTITION: + populateByReflection(command.getEndTxnOnPartition(), traceDetails); + break; + case END_TXN_ON_PARTITION_RESPONSE: + populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails); + break; + case END_TXN_ON_SUBSCRIPTION: + populateByReflection(command.getEndTxnOnSubscription(), traceDetails); + break; + case END_TXN_ON_SUBSCRIPTION_RESPONSE: + populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails); + break; + case TC_CLIENT_CONNECT_REQUEST: + populateByReflection(command.getTcClientConnectRequest(), traceDetails); + break; + case TC_CLIENT_CONNECT_RESPONSE: + populateByReflection(command.getTcClientConnectResponse(), traceDetails); + break; + case WATCH_TOPIC_LIST: + populateByReflection(command.getWatchTopicList(), traceDetails); + break; + case WATCH_TOPIC_LIST_SUCCESS: + populateByReflection(command.getWatchTopicListSuccess(), traceDetails); + break; + case WATCH_TOPIC_UPDATE: + populateByReflection(command.getWatchTopicUpdate(), traceDetails); + break; + case WATCH_TOPIC_LIST_CLOSE: + populateByReflection(command.getWatchTopicListClose(), traceDetails); + break; + case TOPIC_MIGRATED: + populateByReflection(command.getTopicMigrated(), traceDetails); + break; + default: + log.error("Unknown command type: {}", command.getType()); + traceDetails.put("error", "unknownCommandType " + command.getType()); + } + } + + private static void populateByReflection(Object command, Map traceDetails) { + if (command == null) { + return; + } + if (!command.getClass().getCanonicalName().contains("org.apache.pulsar.common.api.proto")) { + return; + } + + Method[] allMethods = command.getClass().getMethods(); + + Arrays.stream(allMethods) + .filter(method -> method.getName().startsWith("has")) + .filter( + method -> { + try { + return (boolean) method.invoke(command); + } catch (Exception e) { + return false; + } + }) + .forEach( + method -> { + String fieldName = method.getName().substring(3); + try { + Optional accessor = + Arrays.stream(allMethods) + .filter( + m -> + m.getName().equals("get" + fieldName) + || m.getName().equals("is" + fieldName)) + .findFirst(); + if (!accessor.isPresent()) { + log.warn( + "No accessor found for field (but has.. counterpart was found): {} of {}", + fieldName, + command.getClass().getCanonicalName()); + return; + } + Object value = accessor.get().invoke(command); + + if (value == null) return; + + // skip logging of binary data + if (value instanceof byte[] + || value instanceof ByteBuf + || value instanceof ByteBuffer) { + int size = 0; + if (value instanceof byte[]) { + size = ((byte[]) value).length; + } else if (value instanceof ByteBuf) { + size = ((ByteBuf) value).readableBytes(); + } else if (value instanceof ByteBuffer) { + size = ((ByteBuffer) value).remaining(); + } + traceDetails.put(fieldName + "_size", size); + return; + } + + if (value + .getClass() + .getCanonicalName() + .contains("org.apache.pulsar.common.api.proto")) { + Map details = new TreeMap<>(); + populateByReflection(value, details); + traceDetails.put(fieldName, details); + } else { + traceDetails.put(fieldName, value); + } + } catch (Exception e) { + log.error("Failed to access field: {}", fieldName, e); + } + }); + } + public static Map getConnectionDetails(TraceLevel level, ServerCnx cnx) { if (cnx == null) { return null; @@ -117,6 +402,8 @@ private static void populateConnectionDetails( ? "no provider" : cnx.getAuthenticationProvider().getAuthMethodName()); break; + case NONE: + break; default: log.warn("Unknown tracing level: {}", level); break; @@ -161,6 +448,8 @@ private static void populateSubscriptionDetails( traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); break; + case NONE: + break; default: log.warn("Unknown tracing level: {}", level); break; @@ -206,6 +495,8 @@ private static void populateConsumerDetails( traceDetails.put("metadata", consumer.getMetadata()); break; + case NONE: + break; default: log.warn("Unknown tracing level: {}", level); break; @@ -254,6 +545,8 @@ private static void populateProducerDetails( } traceDetails.put("remoteCluster", producer.getRemoteCluster()); break; + case NONE: + break; default: log.warn("Unknown tracing level: {}", level); break; @@ -316,6 +609,8 @@ private static void populateMessageMetadataDetails( traceDetails.put("uuid", msgMetadata.getUuid()); } break; + case NONE: + break; default: log.warn("Unknown tracing level: {}", level); break; @@ -353,6 +648,8 @@ private static void populateEntryDetails( traceByteBuf("data", entry.getDataBuffer(), traceDetails); break; + case NONE: + break; default: log.warn("Unknown tracing level: {}", level); break; From f43457ffad0deb17a87fbd190a5afd48888b9d28 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 16 Apr 2024 11:40:26 +0200 Subject: [PATCH 10/29] Fix test --- .../datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java index 4675de18..fa148cc8 100644 --- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -42,21 +42,21 @@ void traceTest() { traces.clear(); trace(mockTracer, "msg", null); assertEquals(1, traces.size()); - assertEquals("{\"message\":\"msg\",\"traceDetails\":null}", traces.get(0)); + assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0)); Map map = new TreeMap<>(); traces.clear(); trace(mockTracer, "msg", map); assertEquals(1, traces.size()); - assertEquals("{\"message\":\"msg\",\"traceDetails\":{}}", traces.get(0)); + assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0)); map.put("key1", "value1"); traces.clear(); trace(mockTracer, "msg", map); assertEquals(1, traces.size()); - assertEquals("{\"message\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); + assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); } // todo: From 1d5e7b87a23943ff8d360b60a23980c851073edc Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 16 Apr 2024 11:45:31 +0200 Subject: [PATCH 11/29] add pulsar-jms-tracing to the release and upgrade the Nifi-plugin --- .github/workflows/release.yml | 2 +- pulsar-jms-admin-ext/pom.xml | 2 +- pulsar-jms-filters/pom.xml | 2 +- pulsar-jms-integration-tests/pom.xml | 4 ++-- pulsar-jms-tracing/pom.xml | 2 +- pulsar-jms/pom.xml | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc9e9cb1..2095b43c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,6 @@ jobs: run: mvn -B package -DskipTests - uses: ncipollo/release-action@v1 with: - artifacts: "pulsar-jms-all/target/pulsar-jms-all-*.jar,resource-adapter/target/pulsarra-rar.rar,pulsar-jms-filters/target/pulsar-jms*.nar,pulsar-jms-cli/target/jms-cli.jar,pulsar-jms-admin-ext/target/pulsar-jms*.nar" + artifacts: "pulsar-jms-all/target/pulsar-jms-all-*.jar,resource-adapter/target/pulsarra-rar.rar,pulsar-jms-filters/target/pulsar-jms*.nar,pulsar-jms-cli/target/jms-cli.jar,pulsar-jms-admin-ext/target/pulsar-jms*.nar,pulsar-jms-tracing/target/pulsar-jms*.nar" token: ${{ secrets.GITHUB_TOKEN }} generateReleaseNotes: true diff --git a/pulsar-jms-admin-ext/pom.xml b/pulsar-jms-admin-ext/pom.xml index d4754733..698c0711 100644 --- a/pulsar-jms-admin-ext/pom.xml +++ b/pulsar-jms-admin-ext/pom.xml @@ -66,7 +66,7 @@ org.apache.nifi nifi-nar-maven-plugin - 1.3.2 + 1.5.1 true pulsar-jms-admin-${project.version} diff --git a/pulsar-jms-filters/pom.xml b/pulsar-jms-filters/pom.xml index 37e22a58..e8e86c98 100644 --- a/pulsar-jms-filters/pom.xml +++ b/pulsar-jms-filters/pom.xml @@ -85,7 +85,7 @@ org.apache.nifi nifi-nar-maven-plugin - 1.3.2 + 1.5.1 true pulsar-jms-${project.version} diff --git a/pulsar-jms-integration-tests/pom.xml b/pulsar-jms-integration-tests/pom.xml index 798d61a8..747bb627 100644 --- a/pulsar-jms-integration-tests/pom.xml +++ b/pulsar-jms-integration-tests/pom.xml @@ -103,8 +103,8 @@ copy filters - - + + diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml index da7d2dda..fc87d02a 100644 --- a/pulsar-jms-tracing/pom.xml +++ b/pulsar-jms-tracing/pom.xml @@ -69,7 +69,7 @@ org.apache.nifi nifi-nar-maven-plugin - 1.3.2 + 1.5.1 true pulsar-jms-tracing-${project.version} diff --git a/pulsar-jms/pom.xml b/pulsar-jms/pom.xml index b715013d..adc3d577 100644 --- a/pulsar-jms/pom.xml +++ b/pulsar-jms/pom.xml @@ -128,10 +128,10 @@ copy filters - - - - + + + + From 7c8750336fe46ebc7e5e89ac84c2e4f25c3ebdcb Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 16 Apr 2024 16:31:24 +0200 Subject: [PATCH 12/29] fix build --- pulsar-jms-tracing/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml index fc87d02a..4c8ff528 100644 --- a/pulsar-jms-tracing/pom.xml +++ b/pulsar-jms-tracing/pom.xml @@ -20,7 +20,7 @@ pulsar-jms-parent com.datastax.oss - 4.1.1-alpha-SNAPSHOT + 4.1.2-alpha-SNAPSHOT 4.0.0 pulsar-jms-tracing From 58f11836c4f06f80f80f5a7ab003605bea0c28c8 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 16 Apr 2024 16:48:49 +0200 Subject: [PATCH 13/29] fix build --- pulsar-jms-tracing/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml index 4c8ff528..ed2bf109 100644 --- a/pulsar-jms-tracing/pom.xml +++ b/pulsar-jms-tracing/pom.xml @@ -20,7 +20,7 @@ pulsar-jms-parent com.datastax.oss - 4.1.2-alpha-SNAPSHOT + 4.1.2-SNAPSHOT 4.0.0 pulsar-jms-tracing From cbe3a5b034e037deaa380f1002a3323c7ccb91c0 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 16 Apr 2024 09:21:31 -0700 Subject: [PATCH 14/29] limit command output, separate event reason option for commands --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 7 +- .../oss/pulsar/jms/tracing/TracingUtils.java | 143 ++++++++++-------- 2 files changed, 86 insertions(+), 64 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 7606563f..a4c44f98 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -62,6 +62,7 @@ public class BrokerTracing implements BrokerInterceptor { public enum EventReasons { ADMINISTRATIVE, + COMMANDS, MESSAGE, TRANSACTION, SERVLET, @@ -269,7 +270,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map } public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { - if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return; if (traceLevel == TraceLevel.NONE) return; @@ -278,7 +279,9 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept if (command.hasType()) { traceDetails.put("type", command.getType().name()); - traceDetails.put("command", getCommandDetails(traceLevel, command)); + if (traceLevel != TraceLevel.MINIMAL) { + traceDetails.put("command", getCommandDetails(traceLevel, command)); + } } else { traceDetails.put("type", "unknown/null"); } diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index cc6ba6de..86334ad6 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -18,12 +18,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -109,181 +111,180 @@ private static void populateCommandDetails( } // trace all params otherwise - switch (command.getType()) { case CONNECT: - populateByReflection(command.getConnect(), traceDetails); + populateByReflection(level, command.getConnect(), traceDetails); break; case CONNECTED: - populateByReflection(command.getConnected(), traceDetails); + populateByReflection(level, command.getConnected(), traceDetails); break; case SUBSCRIBE: - populateByReflection(command.getSubscribe(), traceDetails); + populateByReflection(level, command.getSubscribe(), traceDetails); break; case PRODUCER: - populateByReflection(command.getProducer(), traceDetails); + populateByReflection(level, command.getProducer(), traceDetails); break; case SEND: - populateByReflection(command.getSend(), traceDetails); + populateByReflection(level, command.getSend(), traceDetails); break; case SEND_RECEIPT: - populateByReflection(command.getSendReceipt(), traceDetails); + populateByReflection(level, command.getSendReceipt(), traceDetails); break; case SEND_ERROR: - populateByReflection(command.getSendError(), traceDetails); + populateByReflection(level, command.getSendError(), traceDetails); break; case MESSAGE: - populateByReflection(command.getMessage(), traceDetails); + populateByReflection(level, command.getMessage(), traceDetails); break; case ACK: - populateByReflection(command.getAck(), traceDetails); + populateByReflection(level, command.getAck(), traceDetails); break; case FLOW: - populateByReflection(command.getFlow(), traceDetails); + populateByReflection(level, command.getFlow(), traceDetails); break; case UNSUBSCRIBE: - populateByReflection(command.getUnsubscribe(), traceDetails); + populateByReflection(level, command.getUnsubscribe(), traceDetails); break; case SUCCESS: - populateByReflection(command.getSuccess(), traceDetails); + populateByReflection(level, command.getSuccess(), traceDetails); break; case ERROR: - populateByReflection(command.getError(), traceDetails); + populateByReflection(level, command.getError(), traceDetails); break; case CLOSE_PRODUCER: - populateByReflection(command.getCloseProducer(), traceDetails); + populateByReflection(level, command.getCloseProducer(), traceDetails); break; case CLOSE_CONSUMER: - populateByReflection(command.getCloseConsumer(), traceDetails); + populateByReflection(level, command.getCloseConsumer(), traceDetails); break; case PRODUCER_SUCCESS: - populateByReflection(command.getProducerSuccess(), traceDetails); + populateByReflection(level, command.getProducerSuccess(), traceDetails); break; case PING: - populateByReflection(command.getPing(), traceDetails); + populateByReflection(level, command.getPing(), traceDetails); break; case PONG: - populateByReflection(command.getPong(), traceDetails); + populateByReflection(level, command.getPong(), traceDetails); break; case REDELIVER_UNACKNOWLEDGED_MESSAGES: - populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails); + populateByReflection(level, command.getRedeliverUnacknowledgedMessages(), traceDetails); break; case PARTITIONED_METADATA: - populateByReflection(command.getPartitionMetadata(), traceDetails); + populateByReflection(level, command.getPartitionMetadata(), traceDetails); break; case PARTITIONED_METADATA_RESPONSE: - populateByReflection(command.getPartitionMetadataResponse(), traceDetails); + populateByReflection(level, command.getPartitionMetadataResponse(), traceDetails); break; case LOOKUP: - populateByReflection(command.getLookupTopic(), traceDetails); + populateByReflection(level, command.getLookupTopic(), traceDetails); break; case LOOKUP_RESPONSE: - populateByReflection(command.getLookupTopicResponse(), traceDetails); + populateByReflection(level, command.getLookupTopicResponse(), traceDetails); break; case CONSUMER_STATS: - populateByReflection(command.getConsumerStats(), traceDetails); + populateByReflection(level, command.getConsumerStats(), traceDetails); break; case CONSUMER_STATS_RESPONSE: - populateByReflection(command.getConsumerStatsResponse(), traceDetails); + populateByReflection(level, command.getConsumerStatsResponse(), traceDetails); break; case REACHED_END_OF_TOPIC: - populateByReflection(command.getReachedEndOfTopic(), traceDetails); + populateByReflection(level, command.getReachedEndOfTopic(), traceDetails); break; case SEEK: - populateByReflection(command.getSeek(), traceDetails); + populateByReflection(level, command.getSeek(), traceDetails); break; case GET_LAST_MESSAGE_ID: - populateByReflection(command.getGetLastMessageId(), traceDetails); + populateByReflection(level, command.getGetLastMessageId(), traceDetails); break; case GET_LAST_MESSAGE_ID_RESPONSE: - populateByReflection(command.getGetLastMessageIdResponse(), traceDetails); + populateByReflection(level, command.getGetLastMessageIdResponse(), traceDetails); break; case ACTIVE_CONSUMER_CHANGE: - populateByReflection(command.getActiveConsumerChange(), traceDetails); + populateByReflection(level, command.getActiveConsumerChange(), traceDetails); break; case GET_TOPICS_OF_NAMESPACE: - populateByReflection(command.getGetTopicsOfNamespace(), traceDetails); + populateByReflection(level, command.getGetTopicsOfNamespace(), traceDetails); break; case GET_TOPICS_OF_NAMESPACE_RESPONSE: - populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails); + populateByReflection(level, command.getGetTopicsOfNamespaceResponse(), traceDetails); break; case GET_SCHEMA: - populateByReflection(command.getGetSchema(), traceDetails); + populateByReflection(level, command.getGetSchema(), traceDetails); break; case GET_SCHEMA_RESPONSE: - populateByReflection(command.getGetSchemaResponse(), traceDetails); + populateByReflection(level, command.getGetSchemaResponse(), traceDetails); break; case AUTH_CHALLENGE: - populateByReflection(command.getAuthChallenge(), traceDetails); + populateByReflection(level, command.getAuthChallenge(), traceDetails); break; case AUTH_RESPONSE: - populateByReflection(command.getAuthResponse(), traceDetails); + populateByReflection(level, command.getAuthResponse(), traceDetails); break; case ACK_RESPONSE: - populateByReflection(command.getAckResponse(), traceDetails); + populateByReflection(level, command.getAckResponse(), traceDetails); break; case GET_OR_CREATE_SCHEMA: - populateByReflection(command.getGetOrCreateSchema(), traceDetails); + populateByReflection(level, command.getGetOrCreateSchema(), traceDetails); break; case GET_OR_CREATE_SCHEMA_RESPONSE: - populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails); + populateByReflection(level, command.getGetOrCreateSchemaResponse(), traceDetails); break; case NEW_TXN: - populateByReflection(command.getNewTxn(), traceDetails); + populateByReflection(level, command.getNewTxn(), traceDetails); break; case NEW_TXN_RESPONSE: - populateByReflection(command.getNewTxnResponse(), traceDetails); + populateByReflection(level, command.getNewTxnResponse(), traceDetails); break; case ADD_PARTITION_TO_TXN: - populateByReflection(command.getAddPartitionToTxn(), traceDetails); + populateByReflection(level, command.getAddPartitionToTxn(), traceDetails); break; case ADD_PARTITION_TO_TXN_RESPONSE: - populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails); + populateByReflection(level, command.getAddPartitionToTxnResponse(), traceDetails); break; case ADD_SUBSCRIPTION_TO_TXN: - populateByReflection(command.getAddSubscriptionToTxn(), traceDetails); + populateByReflection(level, command.getAddSubscriptionToTxn(), traceDetails); break; case ADD_SUBSCRIPTION_TO_TXN_RESPONSE: - populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails); + populateByReflection(level, command.getAddSubscriptionToTxnResponse(), traceDetails); break; case END_TXN: - populateByReflection(command.getEndTxn(), traceDetails); + populateByReflection(level, command.getEndTxn(), traceDetails); break; case END_TXN_RESPONSE: - populateByReflection(command.getEndTxnResponse(), traceDetails); + populateByReflection(level, command.getEndTxnResponse(), traceDetails); break; case END_TXN_ON_PARTITION: - populateByReflection(command.getEndTxnOnPartition(), traceDetails); + populateByReflection(level, command.getEndTxnOnPartition(), traceDetails); break; case END_TXN_ON_PARTITION_RESPONSE: - populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails); + populateByReflection(level, command.getEndTxnOnPartitionResponse(), traceDetails); break; case END_TXN_ON_SUBSCRIPTION: - populateByReflection(command.getEndTxnOnSubscription(), traceDetails); + populateByReflection(level, command.getEndTxnOnSubscription(), traceDetails); break; case END_TXN_ON_SUBSCRIPTION_RESPONSE: - populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails); + populateByReflection(level, command.getEndTxnOnSubscriptionResponse(), traceDetails); break; case TC_CLIENT_CONNECT_REQUEST: - populateByReflection(command.getTcClientConnectRequest(), traceDetails); + populateByReflection(level, command.getTcClientConnectRequest(), traceDetails); break; case TC_CLIENT_CONNECT_RESPONSE: - populateByReflection(command.getTcClientConnectResponse(), traceDetails); + populateByReflection(level, command.getTcClientConnectResponse(), traceDetails); break; case WATCH_TOPIC_LIST: - populateByReflection(command.getWatchTopicList(), traceDetails); + populateByReflection(level, command.getWatchTopicList(), traceDetails); break; case WATCH_TOPIC_LIST_SUCCESS: - populateByReflection(command.getWatchTopicListSuccess(), traceDetails); + populateByReflection(level, command.getWatchTopicListSuccess(), traceDetails); break; case WATCH_TOPIC_UPDATE: - populateByReflection(command.getWatchTopicUpdate(), traceDetails); + populateByReflection(level, command.getWatchTopicUpdate(), traceDetails); break; case WATCH_TOPIC_LIST_CLOSE: - populateByReflection(command.getWatchTopicListClose(), traceDetails); + populateByReflection(level, command.getWatchTopicListClose(), traceDetails); break; case TOPIC_MIGRATED: - populateByReflection(command.getTopicMigrated(), traceDetails); + populateByReflection(level, command.getTopicMigrated(), traceDetails); break; default: log.error("Unknown command type: {}", command.getType()); @@ -291,7 +292,18 @@ private static void populateCommandDetails( } } - private static void populateByReflection(Object command, Map traceDetails) { + private static final Set fullTraceFields = + Sets.newHashSet( + "authdata", + "authmethod", + "authmethodname", + "originalauthdata", + "orginalauthmethod", + "originalprincipal", + "schema"); + + private static void populateByReflection( + TraceLevel level, Object command, Map traceDetails) { if (command == null) { return; } @@ -302,7 +314,14 @@ private static void populateByReflection(Object command, Map tra Method[] allMethods = command.getClass().getMethods(); Arrays.stream(allMethods) - .filter(method -> method.getName().startsWith("has")) + .filter( + method -> { + if (!method.getName().startsWith("has")) { + return false; + } + String fieldName = method.getName().substring(3); + return level != TraceLevel.FULL && !fullTraceFields.contains(fieldName.toLowerCase()); + }) .filter( method -> { try { @@ -354,7 +373,7 @@ private static void populateByReflection(Object command, Map tra .getCanonicalName() .contains("org.apache.pulsar.common.api.proto")) { Map details = new TreeMap<>(); - populateByReflection(value, details); + populateByReflection(level, value, details); traceDetails.put(fieldName, details); } else { traceDetails.put(fieldName, value); From 7e480aadcf33b17a29bb1631a4c98e70e0f0c5c5 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 29 Apr 2024 12:04:44 -0700 Subject: [PATCH 15/29] more configuration parameters to reduce verbosity --- .../oss/pulsar/jms/selectors/JMSFilter.java | 3 +- .../oss/pulsar/jms/tracing/BrokerTracing.java | 116 +++++++++++++----- .../oss/pulsar/jms/tracing/TracingUtils.java | 52 ++++---- .../pulsar/jms/tracing/TracingUtilsTest.java | 14 ++- 4 files changed, 126 insertions(+), 59 deletions(-) diff --git a/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java b/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java index dc97f9f5..f8b22834 100644 --- a/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java +++ b/pulsar-jms-filters/src/main/java/com/datastax/oss/pulsar/jms/selectors/JMSFilter.java @@ -68,7 +68,8 @@ public class JMSFilter implements EntryFilter { try { CollectorRegistry.defaultRegistry.register(filterProcessingTime); } catch (IllegalArgumentException alreadyRegistered) { - // this happens in Pulsar 2.10, because each JMSFilter is created in a different classloader + // this happens in Pulsar 2.10, because each JMSFilter is created in a different + // classloader // so the metricRegistered flag doesn't help log.debug("JMSFilter metrics already registered"); } diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index a4c44f98..7a331e69 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.HashSet; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ExecutionException; @@ -72,6 +73,10 @@ public enum EventReasons { private final Set jmsTracingEventList = new HashSet<>(); private TraceLevel traceLevel = defaultTraceLevel; + private int maxBinaryDataLength = 256; + private boolean traceSystemTopics = false; + private boolean traceSchema = false; + private boolean reduceLevelForNestedComponents = true; private static Set loadEnabledEvents( PulsarService pulsarService, Set enabledEvents) { @@ -136,7 +141,7 @@ public TraceLevel load(Subscription sub) { public TraceLevel load(Producer producer) { if (producer.getMetadata() == null || !producer.getMetadata().containsKey("trace")) { - return TraceLevel.NONE; + return TraceLevel.FULL; } try { return TraceLevel.valueOf( @@ -146,7 +151,7 @@ public TraceLevel load(Producer producer) { "Invalid tracing level: {}. Setting to NONE for producer {}", producer.getMetadata().get("trace"), producer); - return TraceLevel.NONE; + return TraceLevel.FULL; } } }); @@ -156,6 +161,21 @@ public void initialize(PulsarService pulsarService) { loadEnabledEvents(pulsarService, jmsTracingEventList); traceLevel = getTraceLevel(pulsarService); + + Properties props = pulsarService.getConfiguration().getProperties(); + if (props.containsKey("jmsTracingMaxBinaryDataLength")) { + maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength")); + } + if (props.containsKey("jmsTracingTraceSystemTopics")) { + traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics")); + } + if (props.containsKey("jmsTracingTraceSchema")) { + traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema")); + } + if (props.containsKey("jmsTracingReduceLevelForNestedComponents")) { + reduceLevelForNestedComponents = + Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents")); + } } @Override @@ -163,14 +183,17 @@ public void close() { log.info("Closing BrokerTracing"); } - private static TraceLevel getTracingLevel(Consumer consumer) { + private TraceLevel getTracingLevel(Consumer consumer) { if (consumer == null) return TraceLevel.NONE; + return getTracingLevel(consumer.getSubscription()); } - private static TraceLevel getTracingLevel(Subscription sub) { + private TraceLevel getTracingLevel(Subscription sub) { if (sub == null) return TraceLevel.NONE; + if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.NONE; + try { return traceLevelForSubscription.get(sub); } catch (ExecutionException e) { @@ -179,9 +202,11 @@ private static TraceLevel getTracingLevel(Subscription sub) { } } - private static TraceLevel getTracingLevel(Producer producer) { + private TraceLevel getTracingLevel(Producer producer) { if (producer == null) return TraceLevel.NONE; + if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.NONE; + try { return traceLevelForProducer.get(producer); } catch (ExecutionException e) { @@ -190,6 +215,13 @@ private static TraceLevel getTracingLevel(Producer producer) { } } + private TraceLevel getTraceLevelForComponent(TraceLevel current) { + if (current == TraceLevel.NONE) return TraceLevel.NONE; + if (reduceLevelForNestedComponents) return TraceLevel.BASIC; + + return current; + } + // private boolean needToTraceTopic(Topic topic) { // if (topic == null) return false; // @@ -218,8 +250,8 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); - traceDetails.put("producer", getProducerDetails(level, producer)); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); + traceDetails.put("producer", getProducerDetails(level, producer, traceSchema)); traceDetails.put("metadata", metadata); trace("Producer created", traceDetails); @@ -232,8 +264,8 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); - traceDetails.put("producer", getProducerDetails(level, producer)); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); + traceDetails.put("producer", getProducerDetails(level, producer, traceSchema)); traceDetails.put("metadata", metadata); trace("Producer closed", traceDetails); @@ -246,7 +278,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); traceDetails.put("consumer", getConsumerDetails(level, consumer)); traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); traceDetails.put("metadata", metadata); @@ -275,7 +307,7 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); if (command.hasType()) { traceDetails.put("type", command.getType().name()); @@ -316,9 +348,10 @@ public void beforeSendMessage( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("subscription", getSubscriptionDetails(level, subscription)); - traceDetails.put("consumer", getConsumerDetails(level, consumer)); - traceDetails.put("entry", getEntryDetails(level, entry)); + traceDetails.put( + "subscription", getSubscriptionDetails(getTraceLevelForComponent(level), subscription)); + traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer)); + traceDetails.put("entry", getEntryDetails(level, entry, maxBinaryDataLength)); traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata)); trace("Before sending message", traceDetails); @@ -333,9 +366,10 @@ public void onMessagePublish( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("producer", getProducerDetails(level, producer)); + traceDetails.put( + "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema)); traceDetails.put("publishContext", getPublishContextDetails(publishContext)); - traceByteBuf("headersAndPayload", headersAndPayload, traceDetails); + traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength); trace("Message publish", traceDetails); } @@ -353,11 +387,11 @@ public void messageProduced( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); - traceDetails.put("producer", getProducerDetails(level, producer)); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); + traceDetails.put( + "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema)); traceDetails.put("publishContext", getPublishContextDetails(publishContext)); - traceDetails.put("ledgerId", ledgerId); - traceDetails.put("entryId", entryId); + traceDetails.put("messageId", ledgerId + ":" + entryId); traceDetails.put("startTimeNs", startTimeNs); trace("Message produced", traceDetails); @@ -371,12 +405,13 @@ public void messageDispatched( if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); - traceDetails.put("consumer", getConsumerDetails(level, consumer)); - traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); - traceDetails.put("ledgerId", ledgerId); - traceDetails.put("entryId", entryId); - traceByteBuf("headersAndPayload", headersAndPayload, traceDetails); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); + traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer)); + traceDetails.put( + "subscription", + getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription())); + traceDetails.put("messageId", ledgerId + ":" + entryId); + traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength); trace("After dispatching message", traceDetails); } @@ -388,13 +423,24 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { if (level == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); - traceDetails.put("consumer", getConsumerDetails(level, consumer)); - traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); + traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer)); + traceDetails.put( + "subscription", + getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription())); Map ackDetails = new TreeMap<>(); - ackDetails.put("type", ackCmd.getAckType().name()); - ackDetails.put("consumerId", ackCmd.getConsumerId()); + if (ackCmd.hasAckType()) { + ackDetails.put("type", ackCmd.getAckType().name()); + } else { + ackDetails.put("type", "NOT SET"); + } + if (ackCmd.hasConsumerId()) { + ackDetails.put("consumerId", ackCmd.getConsumerId()); + } else { + ackDetails.put("consumerId", "NOT SET"); + } + ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount()); ackDetails.put( "messageIds", ackCmd @@ -403,6 +449,14 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { .map(x -> x.getLedgerId() + ":" + x.getEntryId()) .collect(Collectors.toList())); + if (ackCmd.hasTxnidLeastBits() && ackCmd.hasTxnidMostBits()) { + ackDetails.put( + "txnID", "(" + ackCmd.getTxnidMostBits() + "," + ackCmd.getTxnidLeastBits() + ")"); + } + if (ackCmd.hasRequestId()) { + ackDetails.put("requestId", ackCmd.getRequestId()); + } + traceDetails.put("ack", ackDetails); trace("Message acked", traceDetails); diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 86334ad6..87c3ca13 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -39,6 +39,7 @@ import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; @Slf4j public class TracingUtils { @@ -57,8 +58,6 @@ public interface Tracer { .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) .enable(SerializationFeature.WRITE_NULL_MAP_VALUES); - public static final int MAX_DATA_LENGTH = 1024; - public enum TraceLevel { NONE, MINIMAL, @@ -522,18 +521,19 @@ private static void populateConsumerDetails( } } - public static Map getProducerDetails(TraceLevel level, Producer producer) { + public static Map getProducerDetails( + TraceLevel level, Producer producer, boolean traceSchema) { if (producer == null) { return null; } Map details = new TreeMap<>(); - populateProducerDetails(level, producer, details); + populateProducerDetails(level, producer, details, traceSchema); return details; } private static void populateProducerDetails( - TraceLevel level, Producer producer, Map traceDetails) { + TraceLevel level, Producer producer, Map traceDetails, boolean traceSchema) { if (producer == null) { return; } @@ -551,16 +551,25 @@ private static void populateProducerDetails( } break; case BASIC: - populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails); + populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails, traceSchema); traceDetails.put("clientAddress", producer.getClientAddress()); break; case FULL: - populateProducerDetails(TraceLevel.BASIC, producer, traceDetails); + populateProducerDetails(TraceLevel.BASIC, producer, traceDetails, traceSchema); traceDetails.put("metadata", producer.getMetadata()); - if (producer.getSchemaVersion() != null) { - traceDetails.put("schemaVersion", producer.getSchemaVersion().toString()); + + if (traceSchema && producer.getSchemaVersion() != null) { + final String schemaVersion; + if (producer.getSchemaVersion() == SchemaVersion.Empty) { + schemaVersion = "Empty"; + } else if (producer.getSchemaVersion() == SchemaVersion.Latest) { + schemaVersion = "Latest"; + } else { + schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes()); + } + traceDetails.put("schemaVersion", schemaVersion); } traceDetails.put("remoteCluster", producer.getRemoteCluster()); break; @@ -636,36 +645,36 @@ private static void populateMessageMetadataDetails( } } - public static Map getEntryDetails(TraceLevel level, Entry entry) { + public static Map getEntryDetails( + TraceLevel level, Entry entry, int maxBinaryDataLength) { if (entry == null) { return null; } Map details = new TreeMap<>(); - populateEntryDetails(level, entry, details); + populateEntryDetails(level, entry, details, maxBinaryDataLength); return details; } private static void populateEntryDetails( - TraceLevel level, Entry entry, Map traceDetails) { + TraceLevel level, Entry entry, Map traceDetails, int maxBinaryDataLength) { if (entry == null) { return; } switch (level) { case MINIMAL: - traceDetails.put("ledgerId", entry.getLedgerId()); - traceDetails.put("entryId", entry.getEntryId()); + traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId()); break; case BASIC: - populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails); + populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails, maxBinaryDataLength); traceDetails.put("length", entry.getLength()); break; case FULL: - populateEntryDetails(TraceLevel.BASIC, entry, traceDetails); + populateEntryDetails(TraceLevel.BASIC, entry, traceDetails, maxBinaryDataLength); - traceByteBuf("data", entry.getDataBuffer(), traceDetails); + traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength); break; case NONE: break; @@ -699,14 +708,15 @@ private static void populatePublishContext( traceDetails.put("sequenceId", publishContext.getSequenceId()); } - public static void traceByteBuf(String key, ByteBuf buf, Map traceDetails) { - if (buf == null) return; + public static void traceByteBuf( + String key, ByteBuf buf, Map traceDetails, int maxBinaryDataLength) { + if (buf == null || maxBinaryDataLength <= 0) return; - if (buf.readableBytes() < MAX_DATA_LENGTH) { + if (buf.readableBytes() < maxBinaryDataLength) { traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer())); } else { traceDetails.put( - key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, MAX_DATA_LENGTH).nioBuffer())); + key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer())); } } } diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java index fa148cc8..220fee87 100644 --- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -92,27 +92,29 @@ void traceTest() { void traceByteBufTest() { Map traceDetails = new TreeMap<>(); - traceByteBuf("key", null, traceDetails); + int maxBinaryDataLength = 1024; + + traceByteBuf("key", null, traceDetails, maxBinaryDataLength); assertEquals(0, traceDetails.size()); ByteBuf small = Unpooled.buffer(20); for (int i = 0; i < 20; i++) { small.writeByte(i); } - traceByteBuf("key", small, traceDetails); + traceByteBuf("key", small, traceDetails, maxBinaryDataLength); assertEquals(1, traceDetails.size()); assertEquals(42, ((String) traceDetails.get("key")).length()); assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key")); - ByteBuf big = Unpooled.buffer(MAX_DATA_LENGTH + 100); - for (int i = 0; i < MAX_DATA_LENGTH + 100; i++) { + ByteBuf big = Unpooled.buffer(maxBinaryDataLength + 100); + for (int i = 0; i < maxBinaryDataLength + 100; i++) { big.writeByte(i); } traceDetails.clear(); - traceByteBuf("key", big, traceDetails); + traceByteBuf("key", big, traceDetails, maxBinaryDataLength); assertEquals(1, traceDetails.size()); - assertEquals(2 + 2 * MAX_DATA_LENGTH, ((String) traceDetails.get("keySlice")).length()); + assertEquals(2 + 2 * maxBinaryDataLength, ((String) traceDetails.get("keySlice")).length()); assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102")); } } From 0a42c1f50a818d74493e2fd6dc9c55b451847a11 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 29 Apr 2024 14:43:02 -0700 Subject: [PATCH 16/29] updated version to match master / merged poms --- pulsar-jms-tracing/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-jms-tracing/pom.xml b/pulsar-jms-tracing/pom.xml index ed2bf109..19b588df 100644 --- a/pulsar-jms-tracing/pom.xml +++ b/pulsar-jms-tracing/pom.xml @@ -20,7 +20,7 @@ pulsar-jms-parent com.datastax.oss - 4.1.2-SNAPSHOT + 4.1.3-SNAPSHOT 4.0.0 pulsar-jms-tracing From 9c1ad66b0f8fc094876e4855a2c8f51ff98bf7c2 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 29 Apr 2024 15:58:06 -0700 Subject: [PATCH 17/29] more message id details, when available --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 7a331e69..2ab6da2b 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -54,6 +54,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.CommandAck; +import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.intercept.InterceptException; import org.jetbrains.annotations.NotNull; @@ -393,7 +394,6 @@ public void messageProduced( traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceDetails.put("messageId", ledgerId + ":" + entryId); traceDetails.put("startTimeNs", startTimeNs); - trace("Message produced", traceDetails); } @@ -446,7 +446,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { ackCmd .getMessageIdsList() .stream() - .map(x -> x.getLedgerId() + ":" + x.getEntryId()) + .map(BrokerTracing::formatMessageId) .collect(Collectors.toList())); if (ackCmd.hasTxnidLeastBits() && ackCmd.hasTxnidMostBits()) { @@ -462,6 +462,17 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { trace("Message acked", traceDetails); } + @NotNull + private static String formatMessageId(MessageIdData x) { + String msgId = x.getLedgerId() + ":" + x.getEntryId(); + if (x.hasBatchIndex()) { + msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")"; + } else if (x.getAckSetsCount() > 0) { + msgId += " (ackSetCnt " + x.getAckSetsCount() + ")"; + } + return msgId; + } + /* *************************** ** Transaction events ******************************/ From a9d74f8a05e7272d60e9477759563cfee8dbb879 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 30 Apr 2024 15:44:43 -0700 Subject: [PATCH 18/29] Trace levels for producer/consumer can be set via topic properties; a bit more infor for auth is traced --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 121 +++++++++++------- .../oss/pulsar/jms/tracing/TracingUtils.java | 37 +++++- 2 files changed, 112 insertions(+), 46 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 2ab6da2b..98aa617b 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -52,11 +52,13 @@ import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.intercept.InterceptException; +import org.apache.pulsar.common.naming.TopicName; import org.jetbrains.annotations.NotNull; @Slf4j @@ -75,6 +77,7 @@ public enum EventReasons { private final Set jmsTracingEventList = new HashSet<>(); private TraceLevel traceLevel = defaultTraceLevel; private int maxBinaryDataLength = 256; + private int cacheTraceLevelsDurationSec = 10; private boolean traceSystemTopics = false; private boolean traceSchema = false; private boolean reduceLevelForNestedComponents = true; @@ -111,18 +114,20 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) { } } - private static final LoadingCache traceLevelForSubscription = + private final LoadingCache traceLevelForSubscription = CacheBuilder.newBuilder() - .expireAfterWrite(10, TimeUnit.SECONDS) + .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS) .build( new CacheLoader() { public TraceLevel load(Subscription sub) { Map subProps = sub.getSubscriptionProperties(); - if (subProps == null || !subProps.containsKey("trace")) { - return TraceLevel.NONE; - } - try { + if (subProps == null || !subProps.containsKey("trace")) { + PulsarAdmin admin = + sub.getTopic().getBrokerService().getPulsar().getAdminClient(); + return BrokerTracing.getTracingLevel(admin, sub.getTopic()); + } + return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase()); } catch (IllegalArgumentException e) { log.warn( @@ -130,33 +135,55 @@ public TraceLevel load(Subscription sub) { subProps.get("trace"), sub); return TraceLevel.NONE; + } catch (Throwable t) { + log.error("Error getting tracing level", t); + return TraceLevel.NONE; } } }); - private static final LoadingCache traceLevelForProducer = + private final LoadingCache traceLevelForProducer = CacheBuilder.newBuilder() - .expireAfterWrite(10, TimeUnit.SECONDS) + .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS) .build( new CacheLoader() { public TraceLevel load(Producer producer) { - if (producer.getMetadata() == null - || !producer.getMetadata().containsKey("trace")) { - return TraceLevel.FULL; - } try { - return TraceLevel.valueOf( - producer.getMetadata().get("trace").trim().toUpperCase()); - } catch (IllegalArgumentException e) { - log.warn( - "Invalid tracing level: {}. Setting to NONE for producer {}", - producer.getMetadata().get("trace"), - producer); - return TraceLevel.FULL; + PulsarAdmin admin = + producer.getCnx().getBrokerService().getPulsar().getAdminClient(); + Topic topic = producer.getTopic(); + + return BrokerTracing.getTracingLevel(admin, topic); + } catch (Throwable t) { + log.error("Error getting tracing level", t); + return TraceLevel.NONE; } } }); + @NotNull + private static TraceLevel getTracingLevel(PulsarAdmin admin, Topic topic) { + Map props = null; + try { + props = + admin.topics().getProperties(TopicName.get(topic.getName()).getPartitionedTopicName()); + + if (props == null || !props.containsKey("trace")) { + return TraceLevel.NONE; + } + + return TraceLevel.valueOf(props.get("trace").trim().toUpperCase()); + } catch (IllegalArgumentException e) { + log.warn( + "Invalid tracing level: {}. Setting to NONE", + props == null ? "no props" : props.get("trace")); + return TraceLevel.NONE; + } catch (Throwable t) { + log.error("Error getting tracing level", t); + return TraceLevel.NONE; + } + } + public void initialize(PulsarService pulsarService) { log.info("Initializing BrokerTracing"); @@ -177,6 +204,15 @@ public void initialize(PulsarService pulsarService) { reduceLevelForNestedComponents = Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents")); } + if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) { + cacheTraceLevelsDurationSec = + Integer.parseInt(props.getProperty("jmsTracingCacheTraceLevelsDurationSec")); + if (cacheTraceLevelsDurationSec <= 0) { + log.warn( + "Invalid cache duration: {}. Setting to default: {}", cacheTraceLevelsDurationSec, 10); + cacheTraceLevelsDurationSec = 10; + } + } } @Override @@ -223,13 +259,6 @@ private TraceLevel getTraceLevelForComponent(TraceLevel current) { return current; } - // private boolean needToTraceTopic(Topic topic) { - // if (topic == null) return false; - // - // // todo: how to read topic props - // return true; - // } - /* *************************** ** Administrative events ******************************/ @@ -246,13 +275,13 @@ public void onConnectionCreated(ServerCnx cnx) { public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return; - TraceLevel level = getTracingLevel(producer); - if (level == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); - traceDetails.put("producer", getProducerDetails(level, producer, traceSchema)); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); + traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema)); traceDetails.put("metadata", metadata); trace("Producer created", traceDetails); @@ -260,13 +289,13 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return; - TraceLevel level = getTracingLevel(producer); - if (level == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); - traceDetails.put("producer", getProducerDetails(level, producer, traceSchema)); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); + traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema)); traceDetails.put("metadata", metadata); trace("Producer closed", traceDetails); @@ -274,14 +303,15 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return; - TraceLevel level = getTracingLevel(consumer); - if (level == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); - traceDetails.put("consumer", getConsumerDetails(level, consumer)); - traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); + traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer)); + traceDetails.put( + "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription())); traceDetails.put("metadata", metadata); trace("Consumer created", traceDetails); @@ -289,14 +319,15 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return; - TraceLevel level = getTracingLevel(consumer); - if (level == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.NONE) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(level, cnx)); - traceDetails.put("consumer", getConsumerDetails(level, consumer)); - traceDetails.put("subscription", getSubscriptionDetails(level, consumer.getSubscription())); + traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); + traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer)); + traceDetails.put( + "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription())); traceDetails.put("metadata", metadata); trace("Consumer closed", traceDetails); diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 87c3ca13..4e6311df 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -31,6 +31,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; import org.apache.commons.codec.binary.Hex; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Producer; import org.apache.pulsar.broker.service.ServerCnx; @@ -402,6 +403,7 @@ private static void populateConnectionDetails( switch (level) { case MINIMAL: traceDetails.put("clientAddress", cnx.clientAddress().toString()); + traceDetails.put("authRole", cnx.getAuthRole()); break; case BASIC: populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); @@ -412,13 +414,17 @@ private static void populateConnectionDetails( case FULL: populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); - traceDetails.put("authRole", cnx.getAuthRole()); traceDetails.put("authMethod", cnx.getAuthMethod()); traceDetails.put( "authMethodName", cnx.getAuthenticationProvider() == null ? "no provider" : cnx.getAuthenticationProvider().getAuthMethodName()); + + AuthenticationDataSource authData = cnx.getAuthenticationData(); + if (authData != null) { + traceDetails.put("authData", getAuthDataDetails(authData)); + } break; case NONE: break; @@ -428,6 +434,35 @@ private static void populateConnectionDetails( } } + private static Object getAuthDataDetails(AuthenticationDataSource authData) { + if (authData == null) { + return null; + } + + Map details = new TreeMap<>(); + populateAuthDataDetails(authData, details); + return details; + } + + private static void populateAuthDataDetails( + AuthenticationDataSource authData, Map details) { + if (authData == null) { + return; + } + + details.put("peerAddress", authData.getPeerAddress()); + details.put("commandData", authData.getCommandData()); + details.put("httpAuthType", authData.getHttpAuthType()); + details.put("subscription", authData.getSubscription()); + if (authData.getTlsCertificates() != null) { + details.put( + "tlsCertificates", + Arrays.stream(authData.getTlsCertificates()) + .map(Object::toString) + .collect(Collectors.toList())); + } + } + public static Map getSubscriptionDetails(TraceLevel level, Subscription sub) { if (sub == null) { return null; From e910fa6170de5cb2a553b17218d6a61d3d6270a8 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Wed, 1 May 2024 12:40:06 -0700 Subject: [PATCH 19/29] async refresh for caches --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 151 ++++++++++++++---- 1 file changed, 118 insertions(+), 33 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 98aa617b..8a77abee 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -30,6 +30,8 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import io.netty.buffer.ByteBuf; import java.io.IOException; import java.util.HashSet; @@ -37,6 +39,7 @@ import java.util.Properties; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -116,74 +119,156 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) { private final LoadingCache traceLevelForSubscription = CacheBuilder.newBuilder() - .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS) + .maximumSize(10_000L) + .concurrencyLevel(Runtime.getRuntime().availableProcessors()) + .expireAfterWrite(10L * cacheTraceLevelsDurationSec, TimeUnit.SECONDS) + .refreshAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS) .build( new CacheLoader() { public TraceLevel load(Subscription sub) { - Map subProps = sub.getSubscriptionProperties(); - try { - if (subProps == null || !subProps.containsKey("trace")) { - PulsarAdmin admin = - sub.getTopic().getBrokerService().getPulsar().getAdminClient(); - return BrokerTracing.getTracingLevel(admin, sub.getTopic()); - } - - return TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase()); - } catch (IllegalArgumentException e) { - log.warn( - "Invalid tracing level: {}. Setting to NONE for subscription {}", - subProps.get("trace"), - sub); - return TraceLevel.NONE; - } catch (Throwable t) { - log.error("Error getting tracing level", t); - return TraceLevel.NONE; - } + log.info("Loading trace level for subscription {}", sub); + return BrokerTracing.readTraceLevelForSubscription(sub); } - }); + public ListenableFuture reload(Subscription sub, TraceLevel oldValue) + throws Exception { + SettableFuture future = SettableFuture.create(); + BrokerTracing.readTraceLevelForSubscriptionAsync(sub) + .whenComplete( + (level, ex) -> { + if (ex != null) { + future.setException(ex); + } else { + future.set(level); + } + }); + return future; + } + }); private final LoadingCache traceLevelForProducer = CacheBuilder.newBuilder() - .expireAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS) + .maximumSize(10_000L) + .concurrencyLevel(Runtime.getRuntime().availableProcessors()) + .expireAfterWrite(10L * cacheTraceLevelsDurationSec, TimeUnit.SECONDS) + .refreshAfterWrite(cacheTraceLevelsDurationSec, TimeUnit.SECONDS) .build( new CacheLoader() { public TraceLevel load(Producer producer) { try { + log.info("Loading trace level for producer {}", producer); PulsarAdmin admin = producer.getCnx().getBrokerService().getPulsar().getAdminClient(); Topic topic = producer.getTopic(); - return BrokerTracing.getTracingLevel(admin, topic); + return BrokerTracing.readTraceLevelForTopic(admin, topic); } catch (Throwable t) { log.error("Error getting tracing level", t); return TraceLevel.NONE; } } + + public ListenableFuture reload(Producer producer, TraceLevel oldValue) + throws Exception { + SettableFuture future = SettableFuture.create(); + + PulsarAdmin admin = + producer.getCnx().getBrokerService().getPulsar().getAdminClient(); + Topic topic = producer.getTopic(); + + BrokerTracing.readTraceLevelForTopicAsync(admin, topic) + .whenComplete( + (level, ex) -> { + if (ex != null) { + future.setException(ex); + } else { + future.set(level); + } + }); + + return future; + } }); @NotNull - private static TraceLevel getTracingLevel(PulsarAdmin admin, Topic topic) { - Map props = null; + private static TraceLevel readTraceLevelForSubscription(Subscription sub) { try { - props = - admin.topics().getProperties(TopicName.get(topic.getName()).getPartitionedTopicName()); + return readTraceLevelForSubscriptionAsync(sub).get(); + } catch (InterruptedException | ExecutionException e) { + log.error("Interrupted while getting subscription tracing level for {}", sub, e); + Thread.currentThread().interrupt(); + return TraceLevel.NONE; + } catch (Throwable t) { + log.error("Error getting subscription tracing level for {}", sub, t); + return TraceLevel.NONE; + } + } - if (props == null || !props.containsKey("trace")) { - return TraceLevel.NONE; + @NotNull + private static CompletableFuture readTraceLevelForSubscriptionAsync( + Subscription sub) { + Map subProps = sub.getSubscriptionProperties(); + try { + if (subProps == null || !subProps.containsKey("trace")) { + PulsarAdmin admin = sub.getTopic().getBrokerService().getPulsar().getAdminClient(); + return BrokerTracing.readTraceLevelForTopicAsync(admin, sub.getTopic()); } - return TraceLevel.valueOf(props.get("trace").trim().toUpperCase()); + return CompletableFuture.completedFuture( + TraceLevel.valueOf(subProps.get("trace").trim().toUpperCase())); } catch (IllegalArgumentException e) { log.warn( - "Invalid tracing level: {}. Setting to NONE", - props == null ? "no props" : props.get("trace")); + "Invalid tracing level: {}. Setting to NONE for subscription {}", + subProps.get("trace"), + sub); + return CompletableFuture.completedFuture(TraceLevel.NONE); + } catch (Throwable t) { + log.error("Error getting tracing level. Setting to NONE for subscription {}", sub, t); + return CompletableFuture.completedFuture(TraceLevel.NONE); + } + } + + @NotNull + private static TraceLevel readTraceLevelForTopic(PulsarAdmin admin, Topic topic) { + try { + return readTraceLevelForTopicAsync(admin, topic).get(); + } catch (InterruptedException | ExecutionException e) { + log.error("Interrupted while getting tracing level for topic {}", topic.getName(), e); + Thread.currentThread().interrupt(); return TraceLevel.NONE; } catch (Throwable t) { - log.error("Error getting tracing level", t); + log.error("Error getting tracing level for topic {}", topic.getName(), t); return TraceLevel.NONE; } } + @NotNull + private static CompletableFuture readTraceLevelForTopicAsync( + PulsarAdmin admin, Topic topic) { + CompletableFuture> propsFuture = + admin.topics().getPropertiesAsync(TopicName.get(topic.getName()).getPartitionedTopicName()); + return propsFuture.handle( + (props, ex) -> { + if (ex != null) { + log.error("Error getting tracing level for topic {}", topic.getName(), ex); + return TraceLevel.NONE; + } + + try { + if (props == null || !props.containsKey("trace")) { + return TraceLevel.NONE; + } + + return TraceLevel.valueOf(props.get("trace").trim().toUpperCase()); + } catch (IllegalArgumentException e) { + log.warn( + "Invalid tracing level for topic {}: {}. Setting to NONE", + topic.getName(), + props.get("trace")); + return TraceLevel.NONE; + } + }); + } + public void initialize(PulsarService pulsarService) { log.info("Initializing BrokerTracing"); From b1df8d8dea18e3fb1c40e036df2c2b45bbb6b4ef Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Thu, 2 May 2024 16:04:38 -0700 Subject: [PATCH 20/29] part of the feedback addressed, only one level of onfo(still too verbose), better grabularaty for logger --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 207 ++++---- .../oss/pulsar/jms/tracing/TracingUtils.java | 497 ++++++++---------- .../pulsar/jms/tracing/TracingUtilsTest.java | 6 +- 3 files changed, 304 insertions(+), 406 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 8a77abee..68e34beb 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -15,6 +15,7 @@ */ package com.datastax.oss.pulsar.jms.tracing; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventReasons; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails; @@ -34,6 +35,7 @@ import com.google.common.util.concurrent.SettableFuture; import io.netty.buffer.ByteBuf; import java.io.IOException; +import java.util.Arrays; import java.util.HashSet; import java.util.Map; import java.util.Properties; @@ -67,15 +69,7 @@ @Slf4j public class BrokerTracing implements BrokerInterceptor { - public enum EventReasons { - ADMINISTRATIVE, - COMMANDS, - MESSAGE, - TRANSACTION, - SERVLET, - } - - private static final TraceLevel defaultTraceLevel = TraceLevel.BASIC; + private static final TraceLevel defaultTraceLevel = TraceLevel.OFF; private final Set jmsTracingEventList = new HashSet<>(); private TraceLevel traceLevel = defaultTraceLevel; @@ -83,23 +77,26 @@ public enum EventReasons { private int cacheTraceLevelsDurationSec = 10; private boolean traceSystemTopics = false; private boolean traceSchema = false; - private boolean reduceLevelForNestedComponents = true; - private static Set loadEnabledEvents( + private static void loadEnabledEvents( PulsarService pulsarService, Set enabledEvents) { - String events = - pulsarService.getConfiguration().getProperties().getProperty("jmsTracingEventList", ""); - log.debug("read jmsTracingEventList: {}", events); - - for (String event : events.split(",")) { - try { - enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase())); - } catch (IllegalArgumentException e) { - log.error("Invalid event: {}. Skipping", event); + Properties props = pulsarService.getConfiguration().getProperties(); + if (props.contains("jmsTracingEventList")) { + String events = props.getProperty("jmsTracingEventList", ""); + log.debug("read jmsTracingEventList: {}", events); + + enabledEvents.clear(); + for (String event : events.split(",")) { + try { + enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase())); + } catch (IllegalArgumentException e) { + log.error("Invalid event: {}. Skipping", event); + } } + } else { + log.warn("jmsTracingEventList not set. Using all available."); + enabledEvents.addAll(Arrays.asList(EventReasons.values())); } - - return enabledEvents; } @NotNull @@ -163,7 +160,7 @@ public TraceLevel load(Producer producer) { return BrokerTracing.readTraceLevelForTopic(admin, topic); } catch (Throwable t) { log.error("Error getting tracing level", t); - return TraceLevel.NONE; + return TraceLevel.OFF; } } @@ -196,10 +193,10 @@ private static TraceLevel readTraceLevelForSubscription(Subscription sub) { } catch (InterruptedException | ExecutionException e) { log.error("Interrupted while getting subscription tracing level for {}", sub, e); Thread.currentThread().interrupt(); - return TraceLevel.NONE; + return TraceLevel.OFF; } catch (Throwable t) { log.error("Error getting subscription tracing level for {}", sub, t); - return TraceLevel.NONE; + return TraceLevel.OFF; } } @@ -220,10 +217,10 @@ private static CompletableFuture readTraceLevelForSubscriptionAsync( "Invalid tracing level: {}. Setting to NONE for subscription {}", subProps.get("trace"), sub); - return CompletableFuture.completedFuture(TraceLevel.NONE); + return CompletableFuture.completedFuture(TraceLevel.OFF); } catch (Throwable t) { log.error("Error getting tracing level. Setting to NONE for subscription {}", sub, t); - return CompletableFuture.completedFuture(TraceLevel.NONE); + return CompletableFuture.completedFuture(TraceLevel.OFF); } } @@ -234,10 +231,10 @@ private static TraceLevel readTraceLevelForTopic(PulsarAdmin admin, Topic topic) } catch (InterruptedException | ExecutionException e) { log.error("Interrupted while getting tracing level for topic {}", topic.getName(), e); Thread.currentThread().interrupt(); - return TraceLevel.NONE; + return TraceLevel.OFF; } catch (Throwable t) { log.error("Error getting tracing level for topic {}", topic.getName(), t); - return TraceLevel.NONE; + return TraceLevel.OFF; } } @@ -250,12 +247,12 @@ private static CompletableFuture readTraceLevelForTopicAsync( (props, ex) -> { if (ex != null) { log.error("Error getting tracing level for topic {}", topic.getName(), ex); - return TraceLevel.NONE; + return TraceLevel.OFF; } try { if (props == null || !props.containsKey("trace")) { - return TraceLevel.NONE; + return TraceLevel.OFF; } return TraceLevel.valueOf(props.get("trace").trim().toUpperCase()); @@ -264,7 +261,7 @@ private static CompletableFuture readTraceLevelForTopicAsync( "Invalid tracing level for topic {}: {}. Setting to NONE", topic.getName(), props.get("trace")); - return TraceLevel.NONE; + return TraceLevel.OFF; } }); } @@ -285,10 +282,6 @@ public void initialize(PulsarService pulsarService) { if (props.containsKey("jmsTracingTraceSchema")) { traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema")); } - if (props.containsKey("jmsTracingReduceLevelForNestedComponents")) { - reduceLevelForNestedComponents = - Boolean.parseBoolean(props.getProperty("jmsTracingReduceLevelForNestedComponents")); - } if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) { cacheTraceLevelsDurationSec = Integer.parseInt(props.getProperty("jmsTracingCacheTraceLevelsDurationSec")); @@ -306,44 +299,37 @@ public void close() { } private TraceLevel getTracingLevel(Consumer consumer) { - if (consumer == null) return TraceLevel.NONE; + if (consumer == null) return TraceLevel.OFF; return getTracingLevel(consumer.getSubscription()); } private TraceLevel getTracingLevel(Subscription sub) { - if (sub == null) return TraceLevel.NONE; + if (sub == null) return TraceLevel.OFF; - if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.NONE; + if (!traceSystemTopics && sub.getTopic().isSystemTopic()) return TraceLevel.OFF; try { return traceLevelForSubscription.get(sub); } catch (ExecutionException e) { log.error("Error getting tracing level", e); - return TraceLevel.NONE; + return TraceLevel.OFF; } } private TraceLevel getTracingLevel(Producer producer) { - if (producer == null) return TraceLevel.NONE; + if (producer == null) return TraceLevel.OFF; - if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.NONE; + if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return TraceLevel.OFF; try { return traceLevelForProducer.get(producer); } catch (ExecutionException e) { log.error("Error getting tracing level", e); - return TraceLevel.NONE; + return TraceLevel.OFF; } } - private TraceLevel getTraceLevelForComponent(TraceLevel current) { - if (current == TraceLevel.NONE) return TraceLevel.NONE; - if (reduceLevelForNestedComponents) return TraceLevel.BASIC; - - return current; - } - /* *************************** ** Administrative events ******************************/ @@ -351,102 +337,98 @@ private TraceLevel getTraceLevelForComponent(TraceLevel current) { public void onConnectionCreated(ServerCnx cnx) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); - trace("Connection created", traceDetails); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails); } public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); - traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema)); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); - trace("Producer created", traceDetails); + trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails); } public void producerClosed(ServerCnx cnx, Producer producer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); - traceDetails.put("producer", getProducerDetails(traceLevel, producer, traceSchema)); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); - trace("Producer closed", traceDetails); + trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails); } public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); - traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer)); - traceDetails.put( - "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription())); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("consumer", getConsumerDetails(consumer)); + traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("metadata", metadata); - trace("Consumer created", traceDetails); + trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails); } public void consumerClosed(ServerCnx cnx, Consumer consumer, Map metadata) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); - traceDetails.put("consumer", getConsumerDetails(traceLevel, consumer)); - traceDetails.put( - "subscription", getSubscriptionDetails(traceLevel, consumer.getSubscription())); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("consumer", getConsumerDetails(consumer)); + traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("metadata", metadata); - trace("Consumer closed", traceDetails); + trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails); } public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(traceLevel), cnx)); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); if (command.hasType()) { traceDetails.put("type", command.getType().name()); - if (traceLevel != TraceLevel.MINIMAL) { - traceDetails.put("command", getCommandDetails(traceLevel, command)); - } + traceDetails.put("command", getCommandDetails(command)); } else { traceDetails.put("type", "unknown/null"); } - trace("Pulsar command called", traceDetails); + trace(EventReasons.COMMANDS, "Pulsar command called", traceDetails); } public void onConnectionClosed(ServerCnx cnx) { if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(traceLevel, cnx)); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); - trace("Connection closed", traceDetails); + trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails); } /* *************************** @@ -462,16 +444,15 @@ public void beforeSendMessage( if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(subscription); - if (level == TraceLevel.NONE) return; + if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put( - "subscription", getSubscriptionDetails(getTraceLevelForComponent(level), subscription)); - traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer)); - traceDetails.put("entry", getEntryDetails(level, entry, maxBinaryDataLength)); - traceDetails.put("messageMetadata", getMessageMetadataDetails(level, msgMetadata)); + traceDetails.put("subscription", getSubscriptionDetails(subscription)); + traceDetails.put("consumer", getConsumerDetails(consumer)); + traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength)); + traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata)); - trace("Before sending message", traceDetails); + trace(EventReasons.MESSAGE, "Before sending message", traceDetails); } public void onMessagePublish( @@ -480,15 +461,14 @@ public void onMessagePublish( if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(producer); - if (level == TraceLevel.NONE) return; + if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put( - "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema)); + traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength); - trace("Message publish", traceDetails); + trace(EventReasons.MESSAGE, "Message publish", traceDetails); } public void messageProduced( @@ -501,16 +481,15 @@ public void messageProduced( if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(producer); - if (level == TraceLevel.NONE) return; + if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); - traceDetails.put( - "producer", getProducerDetails(getTraceLevelForComponent(level), producer, traceSchema)); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceDetails.put("messageId", ledgerId + ":" + entryId); traceDetails.put("startTimeNs", startTimeNs); - trace("Message produced", traceDetails); + trace(EventReasons.MESSAGE, "Message produced", traceDetails); } public void messageDispatched( @@ -518,32 +497,28 @@ public void messageDispatched( if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(consumer); - if (level == TraceLevel.NONE) return; + if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); - traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer)); - traceDetails.put( - "subscription", - getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription())); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("consumer", getConsumerDetails(consumer)); + traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("messageId", ledgerId + ":" + entryId); traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength); - trace("After dispatching message", traceDetails); + trace(EventReasons.MESSAGE, "After dispatching message", traceDetails); } public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(consumer); - if (level == TraceLevel.NONE) return; + if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(getTraceLevelForComponent(level), cnx)); - traceDetails.put("consumer", getConsumerDetails(getTraceLevelForComponent(level), consumer)); - traceDetails.put( - "subscription", - getSubscriptionDetails(getTraceLevelForComponent(level), consumer.getSubscription())); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("consumer", getConsumerDetails(consumer)); + traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); Map ackDetails = new TreeMap<>(); if (ackCmd.hasAckType()) { @@ -575,7 +550,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { traceDetails.put("ack", ackDetails); - trace("Message acked", traceDetails); + trace(EventReasons.MESSAGE, "Message acked", traceDetails); } @NotNull @@ -595,24 +570,24 @@ private static String formatMessageId(MessageIdData x) { public void txnOpened(long tcId, String txnID) { if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); traceDetails.put("tcId", tcId); traceDetails.put("txnID", txnID); - trace("Transaction opened", traceDetails); + trace(EventReasons.TRANSACTION, "Transaction opened", traceDetails); } public void txnEnded(String txnID, long txnAction) { if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return; - if (traceLevel == TraceLevel.NONE) return; + if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); traceDetails.put("txnID", txnID); traceDetails.put("txnAction", txnAction); - trace("Transaction closed", traceDetails); + trace(EventReasons.TRANSACTION, "Transaction closed", traceDetails); } /* *************************** diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 4e6311df..d995387a 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -23,6 +23,7 @@ import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -44,15 +45,38 @@ @Slf4j public class TracingUtils { - private static final org.slf4j.Logger traceLogger = - org.slf4j.LoggerFactory.getLogger("jms-tracing"); + + public enum EventReasons { + ADMINISTRATIVE, + COMMANDS, + MESSAGE, + TRANSACTION, + SERVLET, + } @FunctionalInterface public interface Tracer { - void trace(String message); + void trace(EventReasons reason, String message); } - public static final Tracer SLF4J_TRACER = traceLogger::info; + public static class Slf4jTracer implements Tracer { + private static final Map traceLoggers = new HashMap<>(); + + static { + for (EventReasons reason : EventReasons.values()) { + traceLoggers.put( + reason, + org.slf4j.LoggerFactory.getLogger("jms-tracing-" + reason.name().toLowerCase())); + } + } + + @Override + public void trace(EventReasons reason, String message) { + traceLoggers.get(reason).info(message); + } + } + + public static final Tracer SLF4J_TRACER = new Slf4jTracer(); private static final ObjectMapper mapper = new ObjectMapper() @@ -60,24 +84,23 @@ public interface Tracer { .enable(SerializationFeature.WRITE_NULL_MAP_VALUES); public enum TraceLevel { - NONE, - MINIMAL, - BASIC, - FULL + OFF, + ON } - public static void trace(String message, Map traceDetails) { - trace(SLF4J_TRACER, message, traceDetails); + public static void trace(EventReasons reason, String message, Map traceDetails) { + trace(SLF4J_TRACER, reason, message, traceDetails); } - public static void trace(Tracer tracer, String message, Map traceDetails) { + public static void trace( + Tracer tracer, EventReasons reason, String message, Map traceDetails) { Map trace = new TreeMap<>(); trace.put("eventType", message); trace.put("traceDetails", traceDetails); try { String loggableJsonString = mapper.writeValueAsString(trace); - tracer.trace(loggableJsonString); + tracer.trace(reason, loggableJsonString); } catch (JsonProcessingException e) { log.error( "Failed to serialize trace event type '{}' as json, traceDetails: {}", @@ -87,24 +110,21 @@ public static void trace(Tracer tracer, String message, Map trac } } - public static Map getCommandDetails(TraceLevel level, BaseCommand command) { + public static Map getCommandDetails(BaseCommand command) { if (command == null) { return null; } Map details = new TreeMap<>(); - populateCommandDetails(level, command, details); + populateCommandDetails(command, details); return details; } private static void populateCommandDetails( - TraceLevel level, BaseCommand command, Map traceDetails) { + BaseCommand command, Map traceDetails) { if (command == null) { return; } - if (level == TraceLevel.NONE) { - return; - } if (!command.hasType()) { return; @@ -113,178 +133,178 @@ private static void populateCommandDetails( // trace all params otherwise switch (command.getType()) { case CONNECT: - populateByReflection(level, command.getConnect(), traceDetails); + populateByReflection(command.getConnect(), traceDetails); break; case CONNECTED: - populateByReflection(level, command.getConnected(), traceDetails); + populateByReflection(command.getConnected(), traceDetails); break; case SUBSCRIBE: - populateByReflection(level, command.getSubscribe(), traceDetails); + populateByReflection(command.getSubscribe(), traceDetails); break; case PRODUCER: - populateByReflection(level, command.getProducer(), traceDetails); + populateByReflection(command.getProducer(), traceDetails); break; case SEND: - populateByReflection(level, command.getSend(), traceDetails); + populateByReflection(command.getSend(), traceDetails); break; case SEND_RECEIPT: - populateByReflection(level, command.getSendReceipt(), traceDetails); + populateByReflection(command.getSendReceipt(), traceDetails); break; case SEND_ERROR: - populateByReflection(level, command.getSendError(), traceDetails); + populateByReflection(command.getSendError(), traceDetails); break; case MESSAGE: - populateByReflection(level, command.getMessage(), traceDetails); + populateByReflection(command.getMessage(), traceDetails); break; case ACK: - populateByReflection(level, command.getAck(), traceDetails); + populateByReflection(command.getAck(), traceDetails); break; case FLOW: - populateByReflection(level, command.getFlow(), traceDetails); + populateByReflection(command.getFlow(), traceDetails); break; case UNSUBSCRIBE: - populateByReflection(level, command.getUnsubscribe(), traceDetails); + populateByReflection(command.getUnsubscribe(), traceDetails); break; case SUCCESS: - populateByReflection(level, command.getSuccess(), traceDetails); + populateByReflection(command.getSuccess(), traceDetails); break; case ERROR: - populateByReflection(level, command.getError(), traceDetails); + populateByReflection(command.getError(), traceDetails); break; case CLOSE_PRODUCER: - populateByReflection(level, command.getCloseProducer(), traceDetails); + populateByReflection(command.getCloseProducer(), traceDetails); break; case CLOSE_CONSUMER: - populateByReflection(level, command.getCloseConsumer(), traceDetails); + populateByReflection(command.getCloseConsumer(), traceDetails); break; case PRODUCER_SUCCESS: - populateByReflection(level, command.getProducerSuccess(), traceDetails); + populateByReflection(command.getProducerSuccess(), traceDetails); break; case PING: - populateByReflection(level, command.getPing(), traceDetails); + populateByReflection(command.getPing(), traceDetails); break; case PONG: - populateByReflection(level, command.getPong(), traceDetails); + populateByReflection(command.getPong(), traceDetails); break; case REDELIVER_UNACKNOWLEDGED_MESSAGES: - populateByReflection(level, command.getRedeliverUnacknowledgedMessages(), traceDetails); + populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails); break; case PARTITIONED_METADATA: - populateByReflection(level, command.getPartitionMetadata(), traceDetails); + populateByReflection(command.getPartitionMetadata(), traceDetails); break; case PARTITIONED_METADATA_RESPONSE: - populateByReflection(level, command.getPartitionMetadataResponse(), traceDetails); + populateByReflection(command.getPartitionMetadataResponse(), traceDetails); break; case LOOKUP: - populateByReflection(level, command.getLookupTopic(), traceDetails); + populateByReflection(command.getLookupTopic(), traceDetails); break; case LOOKUP_RESPONSE: - populateByReflection(level, command.getLookupTopicResponse(), traceDetails); + populateByReflection(command.getLookupTopicResponse(), traceDetails); break; case CONSUMER_STATS: - populateByReflection(level, command.getConsumerStats(), traceDetails); + populateByReflection(command.getConsumerStats(), traceDetails); break; case CONSUMER_STATS_RESPONSE: - populateByReflection(level, command.getConsumerStatsResponse(), traceDetails); + populateByReflection(command.getConsumerStatsResponse(), traceDetails); break; case REACHED_END_OF_TOPIC: - populateByReflection(level, command.getReachedEndOfTopic(), traceDetails); + populateByReflection(command.getReachedEndOfTopic(), traceDetails); break; case SEEK: - populateByReflection(level, command.getSeek(), traceDetails); + populateByReflection(command.getSeek(), traceDetails); break; case GET_LAST_MESSAGE_ID: - populateByReflection(level, command.getGetLastMessageId(), traceDetails); + populateByReflection(command.getGetLastMessageId(), traceDetails); break; case GET_LAST_MESSAGE_ID_RESPONSE: - populateByReflection(level, command.getGetLastMessageIdResponse(), traceDetails); + populateByReflection(command.getGetLastMessageIdResponse(), traceDetails); break; case ACTIVE_CONSUMER_CHANGE: - populateByReflection(level, command.getActiveConsumerChange(), traceDetails); + populateByReflection(command.getActiveConsumerChange(), traceDetails); break; case GET_TOPICS_OF_NAMESPACE: - populateByReflection(level, command.getGetTopicsOfNamespace(), traceDetails); + populateByReflection(command.getGetTopicsOfNamespace(), traceDetails); break; case GET_TOPICS_OF_NAMESPACE_RESPONSE: - populateByReflection(level, command.getGetTopicsOfNamespaceResponse(), traceDetails); + populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails); break; case GET_SCHEMA: - populateByReflection(level, command.getGetSchema(), traceDetails); + populateByReflection(command.getGetSchema(), traceDetails); break; case GET_SCHEMA_RESPONSE: - populateByReflection(level, command.getGetSchemaResponse(), traceDetails); + populateByReflection(command.getGetSchemaResponse(), traceDetails); break; case AUTH_CHALLENGE: - populateByReflection(level, command.getAuthChallenge(), traceDetails); + populateByReflection(command.getAuthChallenge(), traceDetails); break; case AUTH_RESPONSE: - populateByReflection(level, command.getAuthResponse(), traceDetails); + populateByReflection(command.getAuthResponse(), traceDetails); break; case ACK_RESPONSE: - populateByReflection(level, command.getAckResponse(), traceDetails); + populateByReflection(command.getAckResponse(), traceDetails); break; case GET_OR_CREATE_SCHEMA: - populateByReflection(level, command.getGetOrCreateSchema(), traceDetails); + populateByReflection(command.getGetOrCreateSchema(), traceDetails); break; case GET_OR_CREATE_SCHEMA_RESPONSE: - populateByReflection(level, command.getGetOrCreateSchemaResponse(), traceDetails); + populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails); break; case NEW_TXN: - populateByReflection(level, command.getNewTxn(), traceDetails); + populateByReflection(command.getNewTxn(), traceDetails); break; case NEW_TXN_RESPONSE: - populateByReflection(level, command.getNewTxnResponse(), traceDetails); + populateByReflection(command.getNewTxnResponse(), traceDetails); break; case ADD_PARTITION_TO_TXN: - populateByReflection(level, command.getAddPartitionToTxn(), traceDetails); + populateByReflection(command.getAddPartitionToTxn(), traceDetails); break; case ADD_PARTITION_TO_TXN_RESPONSE: - populateByReflection(level, command.getAddPartitionToTxnResponse(), traceDetails); + populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails); break; case ADD_SUBSCRIPTION_TO_TXN: - populateByReflection(level, command.getAddSubscriptionToTxn(), traceDetails); + populateByReflection(command.getAddSubscriptionToTxn(), traceDetails); break; case ADD_SUBSCRIPTION_TO_TXN_RESPONSE: - populateByReflection(level, command.getAddSubscriptionToTxnResponse(), traceDetails); + populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails); break; case END_TXN: - populateByReflection(level, command.getEndTxn(), traceDetails); + populateByReflection(command.getEndTxn(), traceDetails); break; case END_TXN_RESPONSE: - populateByReflection(level, command.getEndTxnResponse(), traceDetails); + populateByReflection(command.getEndTxnResponse(), traceDetails); break; case END_TXN_ON_PARTITION: - populateByReflection(level, command.getEndTxnOnPartition(), traceDetails); + populateByReflection(command.getEndTxnOnPartition(), traceDetails); break; case END_TXN_ON_PARTITION_RESPONSE: - populateByReflection(level, command.getEndTxnOnPartitionResponse(), traceDetails); + populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails); break; case END_TXN_ON_SUBSCRIPTION: - populateByReflection(level, command.getEndTxnOnSubscription(), traceDetails); + populateByReflection(command.getEndTxnOnSubscription(), traceDetails); break; case END_TXN_ON_SUBSCRIPTION_RESPONSE: - populateByReflection(level, command.getEndTxnOnSubscriptionResponse(), traceDetails); + populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails); break; case TC_CLIENT_CONNECT_REQUEST: - populateByReflection(level, command.getTcClientConnectRequest(), traceDetails); + populateByReflection(command.getTcClientConnectRequest(), traceDetails); break; case TC_CLIENT_CONNECT_RESPONSE: - populateByReflection(level, command.getTcClientConnectResponse(), traceDetails); + populateByReflection(command.getTcClientConnectResponse(), traceDetails); break; case WATCH_TOPIC_LIST: - populateByReflection(level, command.getWatchTopicList(), traceDetails); + populateByReflection(command.getWatchTopicList(), traceDetails); break; case WATCH_TOPIC_LIST_SUCCESS: - populateByReflection(level, command.getWatchTopicListSuccess(), traceDetails); + populateByReflection(command.getWatchTopicListSuccess(), traceDetails); break; case WATCH_TOPIC_UPDATE: - populateByReflection(level, command.getWatchTopicUpdate(), traceDetails); + populateByReflection(command.getWatchTopicUpdate(), traceDetails); break; case WATCH_TOPIC_LIST_CLOSE: - populateByReflection(level, command.getWatchTopicListClose(), traceDetails); + populateByReflection(command.getWatchTopicListClose(), traceDetails); break; case TOPIC_MIGRATED: - populateByReflection(level, command.getTopicMigrated(), traceDetails); + populateByReflection(command.getTopicMigrated(), traceDetails); break; default: log.error("Unknown command type: {}", command.getType()); @@ -292,7 +312,7 @@ private static void populateCommandDetails( } } - private static final Set fullTraceFields = + private static final Set skipTraceFields = Sets.newHashSet( "authdata", "authmethod", @@ -302,8 +322,7 @@ private static void populateCommandDetails( "originalprincipal", "schema"); - private static void populateByReflection( - TraceLevel level, Object command, Map traceDetails) { + private static void populateByReflection(Object command, Map traceDetails) { if (command == null) { return; } @@ -320,7 +339,7 @@ private static void populateByReflection( return false; } String fieldName = method.getName().substring(3); - return level != TraceLevel.FULL && !fullTraceFields.contains(fieldName.toLowerCase()); + return !skipTraceFields.contains(fieldName.toLowerCase()); }) .filter( method -> { @@ -356,12 +375,12 @@ private static void populateByReflection( if (value instanceof byte[] || value instanceof ByteBuf || value instanceof ByteBuffer) { - int size = 0; + final int size; if (value instanceof byte[]) { size = ((byte[]) value).length; } else if (value instanceof ByteBuf) { size = ((ByteBuf) value).readableBytes(); - } else if (value instanceof ByteBuffer) { + } else { size = ((ByteBuffer) value).remaining(); } traceDetails.put(fieldName + "_size", size); @@ -373,7 +392,7 @@ private static void populateByReflection( .getCanonicalName() .contains("org.apache.pulsar.common.api.proto")) { Map details = new TreeMap<>(); - populateByReflection(level, value, details); + populateByReflection(value, details); traceDetails.put(fieldName, details); } else { traceDetails.put(fieldName, value); @@ -384,53 +403,35 @@ private static void populateByReflection( }); } - public static Map getConnectionDetails(TraceLevel level, ServerCnx cnx) { + public static Map getConnectionDetails(ServerCnx cnx) { if (cnx == null) { return null; } Map details = new TreeMap<>(); - populateConnectionDetails(level, cnx, details); + populateConnectionDetails(cnx, details); return details; } - private static void populateConnectionDetails( - TraceLevel level, ServerCnx cnx, Map traceDetails) { + private static void populateConnectionDetails(ServerCnx cnx, Map traceDetails) { if (cnx == null) { return; } - switch (level) { - case MINIMAL: - traceDetails.put("clientAddress", cnx.clientAddress().toString()); - traceDetails.put("authRole", cnx.getAuthRole()); - break; - case BASIC: - populateConnectionDetails(TraceLevel.MINIMAL, cnx, traceDetails); - - traceDetails.put("clientVersion", cnx.getClientVersion()); - traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); - break; - case FULL: - populateConnectionDetails(TraceLevel.BASIC, cnx, traceDetails); - - traceDetails.put("authMethod", cnx.getAuthMethod()); - traceDetails.put( - "authMethodName", - cnx.getAuthenticationProvider() == null - ? "no provider" - : cnx.getAuthenticationProvider().getAuthMethodName()); - - AuthenticationDataSource authData = cnx.getAuthenticationData(); - if (authData != null) { - traceDetails.put("authData", getAuthDataDetails(authData)); - } - break; - case NONE: - break; - default: - log.warn("Unknown tracing level: {}", level); - break; + traceDetails.put("clientAddress", cnx.clientAddress()); + traceDetails.put("authRole", cnx.getAuthRole()); + traceDetails.put("clientVersion", cnx.getClientVersion()); + traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); + traceDetails.put("authMethod", cnx.getAuthMethod()); + traceDetails.put( + "authMethodName", + cnx.getAuthenticationProvider() == null + ? "no provider" + : cnx.getAuthenticationProvider().getAuthMethodName()); + + AuthenticationDataSource authData = cnx.getAuthenticationData(); + if (authData != null) { + traceDetails.put("authData", getAuthDataDetails(authData)); } } @@ -463,260 +464,178 @@ private static void populateAuthDataDetails( } } - public static Map getSubscriptionDetails(TraceLevel level, Subscription sub) { + public static Map getSubscriptionDetails(Subscription sub) { if (sub == null) { return null; } Map details = new TreeMap<>(); - populateSubscriptionDetails(level, sub, details); + populateSubscriptionDetails(sub, details); return details; } private static void populateSubscriptionDetails( - TraceLevel level, Subscription sub, Map traceDetails) { + Subscription sub, Map traceDetails) { if (sub == null) { return; } - switch (level) { - case MINIMAL: - traceDetails.put("name", sub.getName()); - traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); - traceDetails.put("type", sub.getType().name()); - break; - case BASIC: - populateSubscriptionDetails(TraceLevel.MINIMAL, sub, traceDetails); - - if (sub.getConsumers() != null) { - traceDetails.put("numberOfConsumers", sub.getConsumers().size()); - traceDetails.put( - "namesOfConsumers", - sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList())); - } + traceDetails.put("name", sub.getName()); + traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); + traceDetails.put("type", sub.getType().name()); - break; - case FULL: - populateSubscriptionDetails(TraceLevel.BASIC, sub, traceDetails); - - traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); - break; - case NONE: - break; - default: - log.warn("Unknown tracing level: {}", level); - break; + if (sub.getConsumers() != null) { + traceDetails.put("numberOfConsumers", sub.getConsumers().size()); + traceDetails.put( + "namesOfConsumers", + sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList())); } + + traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); } - public static Map getConsumerDetails(TraceLevel level, Consumer consumer) { + public static Map getConsumerDetails(Consumer consumer) { if (consumer == null) { return null; } Map details = new TreeMap<>(); - populateConsumerDetails(level, consumer, details); + populateConsumerDetails(consumer, details); return details; } - private static void populateConsumerDetails( - TraceLevel level, Consumer consumer, Map traceDetails) { + private static void populateConsumerDetails(Consumer consumer, Map traceDetails) { if (consumer == null) { return; } - switch (level) { - case MINIMAL: - traceDetails.put("name", consumer.consumerName()); - traceDetails.put("consumerId", consumer.consumerId()); - Subscription sub = consumer.getSubscription(); - if (sub != null) { - traceDetails.put("subscriptionName", sub.getName()); - traceDetails.put( - "topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); - } - break; - case BASIC: - populateConsumerDetails(TraceLevel.MINIMAL, consumer, traceDetails); + traceDetails.put("name", consumer.consumerName()); + traceDetails.put("consumerId", consumer.consumerId()); + Subscription sub = consumer.getSubscription(); + if (sub != null) { + traceDetails.put("subscriptionName", sub.getName()); + traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); + } - traceDetails.put("priorityLevel", consumer.getPriorityLevel()); - traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name()); - traceDetails.put("clientAddress", consumer.getClientAddress()); - break; - case FULL: - populateConsumerDetails(TraceLevel.BASIC, consumer, traceDetails); + traceDetails.put("priorityLevel", consumer.getPriorityLevel()); + traceDetails.put("subType", consumer.subType() == null ? null : consumer.subType().name()); + traceDetails.put("clientAddress", consumer.getClientAddress()); - traceDetails.put("metadata", consumer.getMetadata()); - break; - case NONE: - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } + traceDetails.put("metadata", consumer.getMetadata()); } - public static Map getProducerDetails( - TraceLevel level, Producer producer, boolean traceSchema) { + public static Map getProducerDetails(Producer producer, boolean traceSchema) { if (producer == null) { return null; } Map details = new TreeMap<>(); - populateProducerDetails(level, producer, details, traceSchema); + populateProducerDetails(producer, details, traceSchema); return details; } private static void populateProducerDetails( - TraceLevel level, Producer producer, Map traceDetails, boolean traceSchema) { + Producer producer, Map traceDetails, boolean traceSchema) { if (producer == null) { return; } - switch (level) { - case MINIMAL: - traceDetails.put("producerId", producer.getProducerId()); - traceDetails.put("producerName", producer.getProducerName()); - traceDetails.put( - "accessMode", - producer.getAccessMode() == null ? null : producer.getAccessMode().name()); - if (producer.getTopic() != null) { - traceDetails.put( - "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); - } - break; - case BASIC: - populateProducerDetails(TraceLevel.MINIMAL, producer, traceDetails, traceSchema); + traceDetails.put("producerId", producer.getProducerId()); + traceDetails.put("producerName", producer.getProducerName()); + traceDetails.put( + "accessMode", producer.getAccessMode() == null ? null : producer.getAccessMode().name()); + if (producer.getTopic() != null) { + traceDetails.put( + "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); + } - traceDetails.put("clientAddress", producer.getClientAddress()); - break; - case FULL: - populateProducerDetails(TraceLevel.BASIC, producer, traceDetails, traceSchema); + traceDetails.put("clientAddress", producer.getClientAddress()); - traceDetails.put("metadata", producer.getMetadata()); + traceDetails.put("metadata", producer.getMetadata()); - if (traceSchema && producer.getSchemaVersion() != null) { - final String schemaVersion; - if (producer.getSchemaVersion() == SchemaVersion.Empty) { - schemaVersion = "Empty"; - } else if (producer.getSchemaVersion() == SchemaVersion.Latest) { - schemaVersion = "Latest"; - } else { - schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes()); - } - traceDetails.put("schemaVersion", schemaVersion); - } - traceDetails.put("remoteCluster", producer.getRemoteCluster()); - break; - case NONE: - break; - default: - log.warn("Unknown tracing level: {}", level); - break; + if (traceSchema && producer.getSchemaVersion() != null) { + final String schemaVersion; + if (producer.getSchemaVersion() == SchemaVersion.Empty) { + schemaVersion = "Empty"; + } else if (producer.getSchemaVersion() == SchemaVersion.Latest) { + schemaVersion = "Latest"; + } else { + schemaVersion = "0x" + Hex.encodeHexString(producer.getSchemaVersion().bytes()); + } + traceDetails.put("schemaVersion", schemaVersion); } + traceDetails.put("remoteCluster", producer.getRemoteCluster()); } - public static Map getMessageMetadataDetails( - TraceLevel level, MessageMetadata msgMetadata) { + public static Map getMessageMetadataDetails(MessageMetadata msgMetadata) { if (msgMetadata == null) { return null; } Map details = new TreeMap<>(); - populateMessageMetadataDetails(level, msgMetadata, details); + populateMessageMetadataDetails(msgMetadata, details); return details; } private static void populateMessageMetadataDetails( - TraceLevel level, MessageMetadata msgMetadata, Map traceDetails) { + MessageMetadata msgMetadata, Map traceDetails) { if (msgMetadata == null) { return; } - switch (level) { - case MINIMAL: - if (msgMetadata.hasPartitionKey()) { - traceDetails.put("partitionKey", msgMetadata.getPartitionKey()); - } - if (msgMetadata.hasSequenceId()) { - traceDetails.put("sequenceId", msgMetadata.getSequenceId()); - } - if (msgMetadata.hasProducerName()) { - traceDetails.put("producerName", msgMetadata.getProducerName()); - } - break; - case BASIC: - populateMessageMetadataDetails(TraceLevel.MINIMAL, msgMetadata, traceDetails); - - if (msgMetadata.hasUncompressedSize()) { - traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); - } - if (msgMetadata.hasNumMessagesInBatch()) { - traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); - } - traceDetails.put("serializedSize", msgMetadata.getSerializedSize()); - break; - case FULL: - populateMessageMetadataDetails(TraceLevel.BASIC, msgMetadata, traceDetails); - - if (msgMetadata.hasPublishTime()) { - traceDetails.put("publishTime", msgMetadata.getPublishTime()); - } - if (msgMetadata.hasEventTime()) { - traceDetails.put("eventTime", msgMetadata.getEventTime()); - } - if (msgMetadata.hasReplicatedFrom()) { - traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom()); - } - if (msgMetadata.hasUuid()) { - traceDetails.put("uuid", msgMetadata.getUuid()); - } - break; - case NONE: - break; - default: - log.warn("Unknown tracing level: {}", level); - break; + if (msgMetadata.hasPartitionKey()) { + traceDetails.put("partitionKey", msgMetadata.getPartitionKey()); + } + if (msgMetadata.hasSequenceId()) { + traceDetails.put("sequenceId", msgMetadata.getSequenceId()); + } + if (msgMetadata.hasProducerName()) { + traceDetails.put("producerName", msgMetadata.getProducerName()); + } + + if (msgMetadata.hasUncompressedSize()) { + traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); + } + if (msgMetadata.hasNumMessagesInBatch()) { + traceDetails.put("numMessagesInBatch", msgMetadata.getNumMessagesInBatch()); + } + traceDetails.put("serializedSize", msgMetadata.getSerializedSize()); + + if (msgMetadata.hasPublishTime()) { + traceDetails.put("publishTime", msgMetadata.getPublishTime()); + } + if (msgMetadata.hasEventTime()) { + traceDetails.put("eventTime", msgMetadata.getEventTime()); + } + if (msgMetadata.hasReplicatedFrom()) { + traceDetails.put("replicatedFrom", msgMetadata.getReplicatedFrom()); + } + if (msgMetadata.hasUuid()) { + traceDetails.put("uuid", msgMetadata.getUuid()); } } - public static Map getEntryDetails( - TraceLevel level, Entry entry, int maxBinaryDataLength) { + public static Map getEntryDetails(Entry entry, int maxBinaryDataLength) { if (entry == null) { return null; } Map details = new TreeMap<>(); - populateEntryDetails(level, entry, details, maxBinaryDataLength); + populateEntryDetails(entry, details, maxBinaryDataLength); return details; } private static void populateEntryDetails( - TraceLevel level, Entry entry, Map traceDetails, int maxBinaryDataLength) { + Entry entry, Map traceDetails, int maxBinaryDataLength) { if (entry == null) { return; } - switch (level) { - case MINIMAL: - traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId()); - break; - case BASIC: - populateEntryDetails(TraceLevel.MINIMAL, entry, traceDetails, maxBinaryDataLength); + traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId()); - traceDetails.put("length", entry.getLength()); - break; - case FULL: - populateEntryDetails(TraceLevel.BASIC, entry, traceDetails, maxBinaryDataLength); + traceDetails.put("length", entry.getLength()); - traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength); - break; - case NONE: - break; - default: - log.warn("Unknown tracing level: {}", level); - break; - } + traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength); } public static Map getPublishContextDetails(Topic.PublishContext publishContext) { diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java index 220fee87..fbae35b2 100644 --- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -32,11 +32,15 @@ class TracingUtilsTest { private Tracer mockTracer = new Tracer() { @Override - public void trace(String msg) { + public void trace(EventReasons reason, String msg) { traces.add(msg); } }; + private static void trace(Tracer mockTracer, String msg, Map traceDetails) { + TracingUtils.trace(mockTracer, EventReasons.SERVLET, msg, traceDetails); + } + @Test void traceTest() { traces.clear(); From 8e8c866c6615c4b5a39ef92d594b630e74f1af26 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Thu, 2 May 2024 16:51:21 -0700 Subject: [PATCH 21/29] resolve hostname of the client --- .../oss/pulsar/jms/tracing/TracingUtils.java | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index d995387a..5caa7e36 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -18,9 +18,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.HashMap; @@ -88,6 +93,44 @@ public enum TraceLevel { ON } + private static final LoadingCache ipResolverCache = + CacheBuilder.newBuilder() + .maximumSize(10_000L) + .concurrencyLevel(Runtime.getRuntime().availableProcessors()) + .build( + new CacheLoader() { + public String load(String clientAddress) { + // Dn resolution can be slow in some cases + // and we do not want to create too many requests to DNS, + // so we cache the result + log.info("resolving DNS for {}", clientAddress); + try { + InetAddress address = InetAddress.getByName(clientAddress); + String hostName = address.getCanonicalHostName(); + if (log.isDebugEnabled()) { + log.debug("Resolved DNS for {} to {}", clientAddress, hostName); + } + return hostName; + } catch (UnknownHostException e) { + log.error("Failed to resolve DNS for {}", clientAddress, e); + return clientAddress; + } + } + }); + + public static String hostNameOf(String clientAddress) { + if (clientAddress == null || clientAddress.isEmpty()) { + return "unknown/null"; + } + + try { + return ipResolverCache.get(clientAddress); + } catch (Throwable t) { + log.error("Failed to resolve DNS for {}", clientAddress, t); + return clientAddress; + } + } + public static void trace(EventReasons reason, String message, Map traceDetails) { trace(SLF4J_TRACER, reason, message, traceDetails); } @@ -418,7 +461,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map return; } - traceDetails.put("clientAddress", cnx.clientAddress()); + traceDetails.put("clientAddress", hostNameOf(cnx.clientSourceAddress())); + traceDetails.put("clientSocket", cnx.clientAddress()); traceDetails.put("authRole", cnx.getAuthRole()); traceDetails.put("clientVersion", cnx.getClientVersion()); traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); @@ -519,7 +563,7 @@ private static void populateConsumerDetails(Consumer consumer, Map Date: Thu, 2 May 2024 17:55:09 -0700 Subject: [PATCH 22/29] parse payload out of the ByteBuf --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 6 ++- .../oss/pulsar/jms/tracing/TracingUtils.java | 47 +++++++++++++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 68e34beb..bdcf22ac 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -466,7 +466,11 @@ public void onMessagePublish( Map traceDetails = new TreeMap<>(); traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("publishContext", getPublishContextDetails(publishContext)); - traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength); + + Map headersAndPayloadDetails = new TreeMap<>(); + traceByteBuf( + "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); + traceDetails.put("headersAndPayload", headersAndPayloadDetails); trace(EventReasons.MESSAGE, "Message publish", traceDetails); } diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 5caa7e36..12af1721 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -27,6 +27,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -45,7 +46,10 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.compression.CompressionCodec; +import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaVersion; @Slf4j @@ -709,12 +713,45 @@ private static void populatePublishContext( public static void traceByteBuf( String key, ByteBuf buf, Map traceDetails, int maxBinaryDataLength) { if (buf == null || maxBinaryDataLength <= 0) return; + try { - if (buf.readableBytes() < maxBinaryDataLength) { - traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer())); - } else { - traceDetails.put( - key + "Slice", "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer())); + final ByteBuf metadataAndPayload = buf.retainedDuplicate(); + ByteBuf uncompressedPayload = null; + try { + // advance readerIndex + MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload); + + // todo: do we need to trace this metadata? + populateMessageMetadataDetails(metadata, traceDetails); + + // Decode if needed + CompressionCodec codec = + CompressionCodecProvider.getCompressionCodec(metadata.getCompression()); + uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize()); + + // todo: does this require additional steps if messages are batched? + if (uncompressedPayload.readableBytes() < maxBinaryDataLength + 3) { + String dataAsString = uncompressedPayload.toString(StandardCharsets.UTF_8); + traceDetails.put(key, dataAsString); + } else { + String dataAsString = + uncompressedPayload.toString(0, maxBinaryDataLength, StandardCharsets.UTF_8); + traceDetails.put(key, dataAsString + "..."); + } + } finally { + metadataAndPayload.release(); + if (uncompressedPayload != null) { + uncompressedPayload.release(); + } + } + } catch (Throwable t) { + log.error("Failed to convert ByteBuf to string", t); + if (buf.readableBytes() < maxBinaryDataLength + 3) { + traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer())); + } else { + traceDetails.put( + key, "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()) + "..."); + } } } } From 277e0b45f20b9c4503e14b7efb6680d1c603568e Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Thu, 2 May 2024 18:02:03 -0700 Subject: [PATCH 23/29] fix test, some wording --- .../datastax/oss/pulsar/jms/tracing/BrokerTracing.java | 8 ++++++-- .../com/datastax/oss/pulsar/jms/tracing/TracingUtils.java | 2 +- .../datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index bdcf22ac..dfbfdec7 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -470,7 +470,7 @@ public void onMessagePublish( Map headersAndPayloadDetails = new TreeMap<>(); traceByteBuf( "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); - traceDetails.put("headersAndPayload", headersAndPayloadDetails); + traceDetails.put("payload", headersAndPayloadDetails); trace(EventReasons.MESSAGE, "Message publish", traceDetails); } @@ -508,7 +508,11 @@ public void messageDispatched( traceDetails.put("consumer", getConsumerDetails(consumer)); traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("messageId", ledgerId + ":" + entryId); - traceByteBuf("headersAndPayload", headersAndPayload, traceDetails, maxBinaryDataLength); + + Map headersAndPayloadDetails = new TreeMap<>(); + traceByteBuf( + "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); + traceDetails.put("payload", headersAndPayloadDetails); trace(EventReasons.MESSAGE, "After dispatching message", traceDetails); } diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 12af1721..64e4aa92 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -683,7 +683,7 @@ private static void populateEntryDetails( traceDetails.put("length", entry.getLength()); - traceByteBuf("data", entry.getDataBuffer(), traceDetails, maxBinaryDataLength); + traceByteBuf("payload", entry.getDataBuffer(), traceDetails, maxBinaryDataLength); } public static Map getPublishContextDetails(Topic.PublishContext publishContext) { diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java index fbae35b2..71e01bba 100644 --- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -118,7 +118,7 @@ void traceByteBufTest() { traceDetails.clear(); traceByteBuf("key", big, traceDetails, maxBinaryDataLength); assertEquals(1, traceDetails.size()); - assertEquals(2 + 2 * maxBinaryDataLength, ((String) traceDetails.get("keySlice")).length()); - assertTrue(((String) traceDetails.get("keySlice")).startsWith("0x000102")); + assertEquals(2 + 2 * maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length()); + assertTrue(((String) traceDetails.get("key")).startsWith("0x000102")); } } From 87c640897004ed0538d07e0e00a888258fe65762 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Fri, 3 May 2024 15:02:07 -0700 Subject: [PATCH 24/29] WIP reduce verbosity --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 47 ++++++++++++++----- .../oss/pulsar/jms/tracing/TracingUtils.java | 30 ++++++------ 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index dfbfdec7..0af2d244 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -64,6 +64,9 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.intercept.InterceptException; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; +import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; +import org.apache.pulsar.common.util.DateFormatter; import org.jetbrains.annotations.NotNull; @Slf4j @@ -351,7 +354,6 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); @@ -365,10 +367,17 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); + PublisherStatsImpl stats = producer.getStats(); + traceDetails.put("connectedSince", stats.getConnectedSince()); + traceDetails.put("closedAt", DateFormatter.now()); + traceDetails.put("averageMsgSize", stats.getAverageMsgSize()); + traceDetails.put("msgRateIn", stats.getMsgRateIn()); + traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn()); + // no message count in stats? stats.getCount() is not it + trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails); } @@ -379,7 +388,6 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); traceDetails.put("consumer", getConsumerDetails(consumer)); traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("metadata", metadata); @@ -394,11 +402,22 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map if (traceLevel == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); traceDetails.put("consumer", getConsumerDetails(consumer)); traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("metadata", metadata); + ConsumerStatsImpl stats = consumer.getStats(); + + traceDetails.put("connectedSince", stats.getConnectedSince()); + traceDetails.put("closedAt", DateFormatter.now()); + traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry()); + traceDetails.put("msgRateOut", stats.getMsgRateOut()); + traceDetails.put("msgThroughputOut", stats.getMsgThroughputOut()); + traceDetails.put("msgOutCounter", stats.getMsgOutCounter()); + traceDetails.put("bytesOutCounter", stats.getBytesOutCounter()); + traceDetails.put("unackedMessages", stats.getUnackedMessages()); + traceDetails.put("messageAckRate", stats.getMessageAckRate()); + trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails); } @@ -452,7 +471,7 @@ public void beforeSendMessage( traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength)); traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata)); - trace(EventReasons.MESSAGE, "Before sending message", traceDetails); + trace(EventReasons.MESSAGE, "Message read", traceDetails); } public void onMessagePublish( @@ -472,7 +491,7 @@ public void onMessagePublish( "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); traceDetails.put("payload", headersAndPayloadDetails); - trace(EventReasons.MESSAGE, "Message publish", traceDetails); + trace(EventReasons.MESSAGE, "Message received", traceDetails); } public void messageProduced( @@ -493,7 +512,7 @@ public void messageProduced( traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceDetails.put("messageId", ledgerId + ":" + entryId); traceDetails.put("startTimeNs", startTimeNs); - trace(EventReasons.MESSAGE, "Message produced", traceDetails); + trace(EventReasons.MESSAGE, "Message stored", traceDetails); } public void messageDispatched( @@ -504,17 +523,18 @@ public void messageDispatched( if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); traceDetails.put("consumer", getConsumerDetails(consumer)); - traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); + if (consumer != null) { + traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); + } traceDetails.put("messageId", ledgerId + ":" + entryId); Map headersAndPayloadDetails = new TreeMap<>(); traceByteBuf( - "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); + "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); traceDetails.put("payload", headersAndPayloadDetails); - trace(EventReasons.MESSAGE, "After dispatching message", traceDetails); + trace(EventReasons.MESSAGE, "Message dispatched", traceDetails); } public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { @@ -524,9 +544,10 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); traceDetails.put("consumer", getConsumerDetails(consumer)); - traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); + if (consumer != null) { + traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); + } Map ackDetails = new TreeMap<>(); if (ackCmd.hasAckType()) { diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 64e4aa92..25f933e1 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; @@ -100,7 +101,7 @@ public enum TraceLevel { private static final LoadingCache ipResolverCache = CacheBuilder.newBuilder() .maximumSize(10_000L) - .concurrencyLevel(Runtime.getRuntime().availableProcessors()) + .expireAfterWrite(4, TimeUnit.HOURS) .build( new CacheLoader() { public String load(String clientAddress) { @@ -464,18 +465,15 @@ private static void populateConnectionDetails(ServerCnx cnx, Map if (cnx == null) { return; } - - traceDetails.put("clientAddress", hostNameOf(cnx.clientSourceAddress())); - traceDetails.put("clientSocket", cnx.clientAddress()); + traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress())); traceDetails.put("authRole", cnx.getAuthRole()); + traceDetails.put("principal", cnx.getPrincipal()); traceDetails.put("clientVersion", cnx.getClientVersion()); traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); traceDetails.put("authMethod", cnx.getAuthMethod()); - traceDetails.put( - "authMethodName", - cnx.getAuthenticationProvider() == null - ? "no provider" - : cnx.getAuthenticationProvider().getAuthMethodName()); + if (cnx.getAuthenticationProvider() != null) { + traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); + } AuthenticationDataSource authData = cnx.getAuthenticationData(); if (authData != null) { @@ -534,10 +532,10 @@ private static void populateSubscriptionDetails( if (sub.getConsumers() != null) { traceDetails.put("numberOfConsumers", sub.getConsumers().size()); - traceDetails.put( - "namesOfConsumers", - sub.getConsumers().stream().map(Consumer::consumerName).collect(Collectors.toList())); } + traceDetails.put("isReplicated", sub.isReplicated()); + traceDetails.put("numberOfEntriesDelayed", sub.getNumberOfEntriesDelayed()); + traceDetails.put("numberOfEntriesInBacklog", sub.getNumberOfEntriesInBacklog(false)); traceDetails.put("subscriptionProperties", sub.getSubscriptionProperties()); } @@ -567,9 +565,11 @@ private static void populateConsumerDetails(Consumer consumer, Map getProducerDetails(Producer producer, boolean traceSchema) { @@ -597,7 +597,7 @@ private static void populateProducerDetails( "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); } - traceDetails.put("clientAddress", hostNameOf(producer.getClientAddress())); + traceDetails.put("clientHost", hostNameOf(producer.getClientAddress())); traceDetails.put("metadata", producer.getMetadata()); @@ -613,6 +613,8 @@ private static void populateProducerDetails( traceDetails.put("schemaVersion", schemaVersion); } traceDetails.put("remoteCluster", producer.getRemoteCluster()); + + traceDetails.put("authRole", producer.getCnx().getAuthRole()); } public static Map getMessageMetadataDetails(MessageMetadata msgMetadata) { From 398fa3f490e4988cef4cf28d7121667bec0a7f21 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 6 May 2024 16:28:49 -0700 Subject: [PATCH 25/29] reduced traces verbosity, added some details --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 108 +++++++++++++----- .../oss/pulsar/jms/tracing/TracingUtils.java | 14 +-- 2 files changed, 88 insertions(+), 34 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 0af2d244..5023cec9 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -21,7 +21,6 @@ import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails; -import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getMessageMetadataDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails; @@ -333,6 +332,47 @@ private TraceLevel getTracingLevel(Producer producer) { } } + private static void addMinimumProducerDetails( + Producer producer, Map traceDetails) { + if (producer == null) return; + + traceDetails.put("producerId", producer.getProducerId()); + traceDetails.put("producerName", producer.getProducerName()); + if (producer.getAccessMode() != null) { + traceDetails.put("accessMode", producer.getAccessMode().name()); + } + traceDetails.put("clientHost", TracingUtils.hostNameOf(producer.getClientAddress())); + if (producer.getTopic() != null) { + traceDetails.put( + "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); + } + traceDetails.put("authRole", producer.getCnx().getAuthRole()); + } + + private static void addMinimumConsumerSubscriptionDetails( + Consumer consumer, Map traceDetails) { + if (consumer == null) return; + + addMinimumConsumerSubscriptionDetails(consumer, consumer.getSubscription(), traceDetails); + } + + private static void addMinimumConsumerSubscriptionDetails( + Consumer consumer, Subscription subscription, Map traceDetails) { + if (consumer != null) { + traceDetails.put("consumerName", consumer.consumerName()); + traceDetails.put("consumerId", consumer.consumerId()); + traceDetails.put("clientHost", TracingUtils.hostNameOf(consumer.getClientAddress())); + traceDetails.put("authRole", consumer.cnx().getAuthRole()); + } + + if (subscription != null) { + traceDetails.put("subscriptionName", subscription.getName()); + traceDetails.put( + "topicName", TopicName.get(subscription.getTopicName()).getPartitionedTopicName()); + traceDetails.put("subscriptionType", subscription.getType().name()); + } + } + /* *************************** ** Administrative events ******************************/ @@ -344,6 +384,8 @@ public void onConnectionCreated(ServerCnx cnx) { Map traceDetails = new TreeMap<>(); traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); + trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails); } @@ -356,6 +398,7 @@ public void producerCreated(ServerCnx cnx, Producer producer, Map traceDetails = new TreeMap<>(); traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); + traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails); } @@ -369,6 +412,7 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map Map traceDetails = new TreeMap<>(); traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); + traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); PublisherStatsImpl stats = producer.getStats(); traceDetails.put("connectedSince", stats.getConnectedSince()); @@ -391,6 +435,7 @@ public void consumerCreated(ServerCnx cnx, Consumer consumer, Map traceDetails.put("consumer", getConsumerDetails(consumer)); traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("metadata", metadata); + traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); ConsumerStatsImpl stats = consumer.getStats(); - traceDetails.put("connectedSince", stats.getConnectedSince()); traceDetails.put("closedAt", DateFormatter.now()); traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry()); @@ -417,6 +462,7 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map traceDetails.put("bytesOutCounter", stats.getBytesOutCounter()); traceDetails.put("unackedMessages", stats.getUnackedMessages()); traceDetails.put("messageAckRate", stats.getMessageAckRate()); + traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver()); trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails); } @@ -446,6 +492,7 @@ public void onConnectionClosed(ServerCnx cnx) { Map traceDetails = new TreeMap<>(); traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails); } @@ -466,12 +513,12 @@ public void beforeSendMessage( if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("subscription", getSubscriptionDetails(subscription)); - traceDetails.put("consumer", getConsumerDetails(consumer)); + + addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails); + traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength)); - traceDetails.put("messageMetadata", getMessageMetadataDetails(msgMetadata)); - trace(EventReasons.MESSAGE, "Message read", traceDetails); + trace(EventReasons.MESSAGE, "read", traceDetails); } public void onMessagePublish( @@ -483,7 +530,9 @@ public void onMessagePublish( if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("producer", getProducerDetails(producer, traceSchema)); + + addMinimumProducerDetails(producer, traceDetails); + traceDetails.put("publishContext", getPublishContextDetails(publishContext)); Map headersAndPayloadDetails = new TreeMap<>(); @@ -491,7 +540,7 @@ public void onMessagePublish( "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); traceDetails.put("payload", headersAndPayloadDetails); - trace(EventReasons.MESSAGE, "Message received", traceDetails); + trace(EventReasons.MESSAGE, "received", traceDetails); } public void messageProduced( @@ -507,12 +556,12 @@ public void messageProduced( if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); - traceDetails.put("producer", getProducerDetails(producer, traceSchema)); + addMinimumProducerDetails(producer, traceDetails); + traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceDetails.put("messageId", ledgerId + ":" + entryId); traceDetails.put("startTimeNs", startTimeNs); - trace(EventReasons.MESSAGE, "Message stored", traceDetails); + trace(EventReasons.MESSAGE, "stored", traceDetails); } public void messageDispatched( @@ -523,10 +572,7 @@ public void messageDispatched( if (level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("consumer", getConsumerDetails(consumer)); - if (consumer != null) { - traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); - } + addMinimumConsumerSubscriptionDetails(consumer, traceDetails); traceDetails.put("messageId", ledgerId + ":" + entryId); Map headersAndPayloadDetails = new TreeMap<>(); @@ -534,31 +580,41 @@ public void messageDispatched( "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); traceDetails.put("payload", headersAndPayloadDetails); - trace(EventReasons.MESSAGE, "Message dispatched", traceDetails); + trace(EventReasons.MESSAGE, "dispatched", traceDetails); } public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; TraceLevel level = getTracingLevel(consumer); - if (level == TraceLevel.OFF) return; + if (consumer != null && level == TraceLevel.OFF) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("consumer", getConsumerDetails(consumer)); - if (consumer != null) { - traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); + + addMinimumConsumerSubscriptionDetails(consumer, traceDetails); + + if (consumer == null) { + // ack with empty consumer == message filtered by JMSFilter + traceDetails.put("reason", "filtered by JMSFilter"); + } else { + // todo: am I right that unacked/nacked messages never go through broker interceptor? + // in this case we need consumer interceptor to track nacks + traceDetails.put("reason", "acked"); + } + + if (consumer != null && consumer.getSubscription() != null) { + Subscription sub = consumer.getSubscription(); + traceDetails.put("subscriptionName", sub.getName()); + traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); + traceDetails.put("subscriptionType", sub.getType().name()); } Map ackDetails = new TreeMap<>(); if (ackCmd.hasAckType()) { ackDetails.put("type", ackCmd.getAckType().name()); - } else { - ackDetails.put("type", "NOT SET"); } if (ackCmd.hasConsumerId()) { - ackDetails.put("consumerId", ackCmd.getConsumerId()); - } else { - ackDetails.put("consumerId", "NOT SET"); + ackDetails.put("ackConsumerId", ackCmd.getConsumerId()); } ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount()); ackDetails.put( @@ -579,7 +635,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { traceDetails.put("ack", ackDetails); - trace(EventReasons.MESSAGE, "Message acked", traceDetails); + trace(EventReasons.MESSAGE, "acknowledged", traceDetails); } @NotNull diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 25f933e1..f0317a0c 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -475,6 +475,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map traceDetails.put("authMethodName", cnx.getAuthenticationProvider().getAuthMethodName()); } + traceDetails.put("state", cnx.getState()); + AuthenticationDataSource authData = cnx.getAuthenticationData(); if (authData != null) { traceDetails.put("authData", getAuthDataDetails(authData)); @@ -639,10 +641,6 @@ private static void populateMessageMetadataDetails( if (msgMetadata.hasSequenceId()) { traceDetails.put("sequenceId", msgMetadata.getSequenceId()); } - if (msgMetadata.hasProducerName()) { - traceDetails.put("producerName", msgMetadata.getProducerName()); - } - if (msgMetadata.hasUncompressedSize()) { traceDetails.put("uncompressedSize", msgMetadata.getUncompressedSize()); } @@ -703,12 +701,12 @@ private static void populatePublishContext( traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage()); traceDetails.put("isChunked", publishContext.isChunked()); traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages()); - traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp()); traceDetails.put("msgSize", publishContext.getMsgSize()); - traceDetails.put("producerName", publishContext.getProducerName()); - traceDetails.put("originalProducerName", publishContext.getOriginalProducerName()); - traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId()); + if (publishContext.getOriginalProducerName() != null) { + traceDetails.put("originalProducerName", publishContext.getOriginalProducerName()); + traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId()); + } traceDetails.put("sequenceId", publishContext.getSequenceId()); } From c74a95bbe3ac5f8d7f029d499581dbbbc1db850b Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 7 May 2024 18:31:12 -0700 Subject: [PATCH 26/29] tweaks, servlet tracing --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 121 +++++++++++++----- .../oss/pulsar/jms/tracing/TracingUtils.java | 76 +++++++++++ 2 files changed, 168 insertions(+), 29 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index 5023cec9..cc34cf57 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -24,6 +24,7 @@ import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getProducerDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getPublishContextDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.hostNameOf; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf; @@ -33,7 +34,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.netty.buffer.ByteBuf; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Map; @@ -44,11 +44,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.service.Consumer; @@ -83,9 +85,9 @@ public class BrokerTracing implements BrokerInterceptor { private static void loadEnabledEvents( PulsarService pulsarService, Set enabledEvents) { Properties props = pulsarService.getConfiguration().getProperties(); - if (props.contains("jmsTracingEventList")) { + if (props.containsKey("jmsTracingEventList")) { String events = props.getProperty("jmsTracingEventList", ""); - log.debug("read jmsTracingEventList: {}", events); + log.info("read jmsTracingEventList: {}", events); enabledEvents.clear(); for (String event : events.split(",")) { @@ -109,7 +111,9 @@ private static TraceLevel getTraceLevel(PulsarService pulsar) { .getProperties() .getProperty("jmsTracingLevel", defaultTraceLevel.toString()); try { - return TraceLevel.valueOf(level.trim().toUpperCase()); + TraceLevel traceLevel = TraceLevel.valueOf(level.trim().toUpperCase()); + log.info("Using tracing level: {}", traceLevel); + return traceLevel; } catch (IllegalArgumentException e) { log.warn("Invalid tracing level: {}. Using default: {}", level, defaultTraceLevel); return defaultTraceLevel; @@ -268,6 +272,17 @@ private static CompletableFuture readTraceLevelForTopicAsync( }); } + @NotNull + private static String formatMessageId(MessageIdData x) { + String msgId = x.getLedgerId() + ":" + x.getEntryId(); + if (x.hasBatchIndex()) { + msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")"; + } else if (x.getAckSetsCount() > 0) { + msgId += " (ackSetCnt " + x.getAckSetsCount() + ")"; + } + return msgId; + } + public void initialize(PulsarService pulsarService) { log.info("Initializing BrokerTracing"); @@ -277,12 +292,15 @@ public void initialize(PulsarService pulsarService) { Properties props = pulsarService.getConfiguration().getProperties(); if (props.containsKey("jmsTracingMaxBinaryDataLength")) { maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength")); + log.info("Setting maxBinaryDataLength to {}", maxBinaryDataLength); } if (props.containsKey("jmsTracingTraceSystemTopics")) { traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics")); + log.info("Setting traceSystemTopics to {}", traceSystemTopics); } if (props.containsKey("jmsTracingTraceSchema")) { traceSchema = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSchema")); + log.info("Setting traceSchema to {}", traceSchema); } if (props.containsKey("jmsTracingCacheTraceLevelsDurationSec")) { cacheTraceLevelsDurationSec = @@ -292,6 +310,7 @@ public void initialize(PulsarService pulsarService) { "Invalid cache duration: {}. Setting to default: {}", cacheTraceLevelsDurationSec, 10); cacheTraceLevelsDurationSec = 10; } + log.info("Setting cacheTraceLevelsDurationSec to {}", cacheTraceLevelsDurationSec); } } @@ -472,12 +491,47 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws Intercept if (traceLevel == TraceLevel.OFF) return; + traceCommandInternal(command, cnx); + + // todo: cache topics, and check if system topic + // without cache: + + // final String topicName = getCommandTopic(command); + // if (topicName == null) { + // // don't know if a system topic, trace it + // traceCommandInternal(command, cnx); + // return; + // } + // + // cnx.getBrokerService() + // .getTopicIfExists(topicName) + // .whenComplete( + // (topic, ex) -> { + // if (ex != null) { + // log.error("Error getting topic {} to trace command {}", topicName, command, + // ex); + // return; + // } + // + // // skip system topics if needed + // if (!traceSystemTopics && topic.isPresent() && topic.get().isSystemTopic()) + // return; + // + // traceCommandInternal(command, cnx); + // }); + } + + private static void traceCommandInternal(BaseCommand command, ServerCnx cnx) { Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); + + traceDetails.put("authMethod", cnx.getAuthMethod()); + traceDetails.put("authRole", cnx.getAuthRole()); + traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress())); + traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); if (command.hasType()) { - traceDetails.put("type", command.getType().name()); - traceDetails.put("command", getCommandDetails(command)); + traceDetails.put("command", command.getType().name()); + traceDetails.put("parameters", getCommandDetails(command)); } else { traceDetails.put("type", "unknown/null"); } @@ -638,17 +692,6 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { trace(EventReasons.MESSAGE, "acknowledged", traceDetails); } - @NotNull - private static String formatMessageId(MessageIdData x) { - String msgId = x.getLedgerId() + ":" + x.getEntryId(); - if (x.hasBatchIndex()) { - msgId += " (batchSize: " + x.getBatchSize() + "|ackSetCnt: " + x.getAckSetsCount() + ")"; - } else if (x.getAckSetsCount() > 0) { - msgId += " (ackSetCnt " + x.getAckSetsCount() + ")"; - } - return msgId; - } - /* *************************** ** Transaction events ******************************/ @@ -679,18 +722,38 @@ public void txnEnded(String txnID, long txnAction) { ** Servlet events ******************************/ - public void onWebserviceRequest(ServletRequest request) - throws IOException, ServletException, InterceptException { - // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) { - // log.info("onWebserviceRequest: Tracing servlet requests not supported"); - // } + public void onWebserviceRequest(ServletRequest request) { + // skipping, it is the same as onWebserviceResponse + // but without response status. } - public void onWebserviceResponse(ServletRequest request, ServletResponse response) - throws IOException, ServletException { - // if (getEnabledEvents(???).contains(EventReasons.SERVLET)) { - // log.info("onWebserviceResponse: Tracing servlet requests not supported"); - // } + public void onWebserviceResponse(ServletRequest request, ServletResponse response) { + if (!jmsTracingEventList.contains(EventReasons.SERVLET)) return; + if (traceLevel == TraceLevel.OFF) return; + + Map traceDetails = new TreeMap<>(); + + traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost())); + traceDetails.put("protocol", request.getProtocol()); + traceDetails.put("scheme", request.getScheme()); + + try { + HttpServletRequest req = (HttpServletRequest) FieldUtils.readField(request, "request", true); + traceDetails.put("method", req.getMethod()); + traceDetails.put("uri", req.getRequestURI()); + if (req.getQueryString() != null) { + traceDetails.put("queryString", req.getQueryString()); + } + traceDetails.put("authType", req.getAuthType()); + traceDetails.put("remoteUser", req.getRemoteUser()); + + HttpServletResponse resp = (HttpServletResponse) response; + traceDetails.put("status", resp.getStatus()); + } catch (Throwable t) { + log.error("Error getting request details", t); + } + + trace(EventReasons.SERVLET, "WebService response", traceDetails); } // not needed diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index f0317a0c..136ad5fd 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -360,6 +360,82 @@ private static void populateCommandDetails( } } + public static String getCommandTopic(BaseCommand command) { + if (command == null) { + return null; + } + + if (!command.hasType()) { + return null; + } + + // Currently not doing transitive topic resolution by e.g. topic pattern + // or by producerId/consumerId to the topic they are using. + // Also, _RESPONSE counterparts do not have topic, and we aren't matching to the request's by + // requestId. + switch (command.getType()) { + case SUBSCRIBE: + if (command.getSubscribe().hasTopic()) { + return command.getSubscribe().getTopic(); + } + break; + case PRODUCER: + if (command.getProducer().hasTopic()) { + return command.getProducer().getTopic(); + } + break; + case PARTITIONED_METADATA: + if (command.getPartitionMetadata().hasTopic()) { + return command.getPartitionMetadata().getTopic(); + } + break; + case LOOKUP: + if (command.getLookupTopic().hasTopic()) { + return command.getLookupTopic().getTopic(); + } + break; + case GET_SCHEMA: + if (command.getGetSchema().hasTopic()) { + return command.getGetSchema().getTopic(); + } + break; + case GET_OR_CREATE_SCHEMA: + if (command.getGetOrCreateSchema().hasTopic()) { + return command.getGetOrCreateSchema().getTopic(); + } + break; + case ADD_SUBSCRIPTION_TO_TXN: + if (command.getAddSubscriptionToTxn().getSubscriptionsCount() > 0 + && command.getAddSubscriptionToTxn().getSubscriptionsList().get(0).hasTopic()) { + Optional subscription = + command + .getAddSubscriptionToTxn() + .getSubscriptionsList() + .stream() + .filter(sub -> sub.hasTopic()) + .findFirst(); + if (subscription.isPresent()) { + return subscription.get().getTopic(); + } + } + break; + case END_TXN_ON_PARTITION: + if (command.getEndTxnOnPartition().hasTopic()) { + return command.getEndTxnOnPartition().getTopic(); + } + break; + case END_TXN_ON_SUBSCRIPTION: + if (command.getEndTxnOnSubscription().hasSubscription() + && command.getEndTxnOnSubscription().getSubscription().hasTopic()) { + return command.getEndTxnOnSubscription().getSubscription().getTopic(); + } + break; + default: + return null; + } + return null; + } + private static final Set skipTraceFields = Sets.newHashSet( "authdata", From 70a82888827f8e98a16a1bed682c0dc70a83ba7e Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Wed, 8 May 2024 14:24:48 -0700 Subject: [PATCH 27/29] jmsTracingLevel as a master off switch; reworked categories, renamed config param --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 266 +++++------ .../oss/pulsar/jms/tracing/TracingUtils.java | 440 ++---------------- .../pulsar/jms/tracing/TracingUtilsTest.java | 12 +- 3 files changed, 172 insertions(+), 546 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index cc34cf57..dafa8ee0 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -15,9 +15,9 @@ */ package com.datastax.oss.pulsar.jms.tracing; -import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventReasons; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventCategory; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.EventSubCategory; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.TraceLevel; -import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getCommandDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConnectionDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getConsumerDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getEntryDetails; @@ -34,7 +34,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.netty.buffer.ByteBuf; -import java.util.Arrays; import java.util.HashSet; import java.util.Map; import java.util.Properties; @@ -63,7 +62,7 @@ import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.intercept.InterceptException; +import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; @@ -75,7 +74,7 @@ public class BrokerTracing implements BrokerInterceptor { private static final TraceLevel defaultTraceLevel = TraceLevel.OFF; - private final Set jmsTracingEventList = new HashSet<>(); + private final Set jmsTracingEventCategory = new HashSet<>(); private TraceLevel traceLevel = defaultTraceLevel; private int maxBinaryDataLength = 256; private int cacheTraceLevelsDurationSec = 10; @@ -83,23 +82,30 @@ public class BrokerTracing implements BrokerInterceptor { private boolean traceSchema = false; private static void loadEnabledEvents( - PulsarService pulsarService, Set enabledEvents) { + PulsarService pulsarService, Set enabledEvents, TraceLevel traceLevel) { + + if (traceLevel == TraceLevel.OFF) { + log.info("Tracing is disabled. jmsTracingEventCategory is ignored."); + enabledEvents.clear(); + return; + } + Properties props = pulsarService.getConfiguration().getProperties(); - if (props.containsKey("jmsTracingEventList")) { - String events = props.getProperty("jmsTracingEventList", ""); - log.info("read jmsTracingEventList: {}", events); + if (props.containsKey("jmsTracingEventCategory")) { + String events = props.getProperty("jmsTracingEventCategory", ""); + log.info("read jmsTracingEventCategory: {}", events); enabledEvents.clear(); for (String event : events.split(",")) { try { - enabledEvents.add(EventReasons.valueOf(event.trim().toUpperCase())); + enabledEvents.add(EventCategory.valueOf(event.trim().toUpperCase())); } catch (IllegalArgumentException e) { log.error("Invalid event: {}. Skipping", event); } } } else { - log.warn("jmsTracingEventList not set. Using all available."); - enabledEvents.addAll(Arrays.asList(EventReasons.values())); + log.warn("jmsTracingEventCategory is not set. Using ADMIN, CONN."); + enabledEvents.add(EventCategory.CONN); } } @@ -286,9 +292,10 @@ private static String formatMessageId(MessageIdData x) { public void initialize(PulsarService pulsarService) { log.info("Initializing BrokerTracing"); - loadEnabledEvents(pulsarService, jmsTracingEventList); traceLevel = getTraceLevel(pulsarService); + loadEnabledEvents(pulsarService, jmsTracingEventCategory, traceLevel); + Properties props = pulsarService.getConfiguration().getProperties(); if (props.containsKey("jmsTracingMaxBinaryDataLength")) { maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength")); @@ -393,41 +400,49 @@ private static void addMinimumConsumerSubscriptionDetails( } /* *************************** - ** Administrative events + ** Connection events ******************************/ public void onConnectionCreated(ServerCnx cnx) { - if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.CONN)) return; + + Map traceDetails = new TreeMap<>(); + traceDetails.put("serverCnx", getConnectionDetails(cnx)); + traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); - if (traceLevel == TraceLevel.OFF) return; + trace(EventCategory.CONN, EventSubCategory.CREATED, traceDetails); + } + + public void onConnectionClosed(ServerCnx cnx) { + if (!jmsTracingEventCategory.contains(EventCategory.CONN)) return; Map traceDetails = new TreeMap<>(); traceDetails.put("serverCnx", getConnectionDetails(cnx)); traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); - trace(EventReasons.ADMINISTRATIVE, "Connection created", traceDetails); + trace(EventCategory.CONN, EventSubCategory.CLOSED, traceDetails); } + /* ************************** + * Producer connection events + *****************************/ + public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { - if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.PROD)) return; if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.OFF) return; - Map traceDetails = new TreeMap<>(); traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); - trace(EventReasons.ADMINISTRATIVE, "Producer created", traceDetails); + trace(EventCategory.PROD, EventSubCategory.CREATED, traceDetails); } public void producerClosed(ServerCnx cnx, Producer producer, Map metadata) { - if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.PROD)) return; if (!traceSystemTopics && producer.getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.OFF) return; - Map traceDetails = new TreeMap<>(); traceDetails.put("producer", getProducerDetails(producer, traceSchema)); traceDetails.put("metadata", metadata); @@ -441,30 +456,30 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn()); // no message count in stats? stats.getCount() is not it - trace(EventReasons.ADMINISTRATIVE, "Producer closed", traceDetails); + trace(EventCategory.PROD, EventSubCategory.CLOSED, traceDetails); } + /* ************************** + * Consumer connection events + *****************************/ + public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { - if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.CONS)) return; if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.OFF) return; - Map traceDetails = new TreeMap<>(); traceDetails.put("consumer", getConsumerDetails(consumer)); traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); traceDetails.put("metadata", metadata); traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); - trace(EventReasons.ADMINISTRATIVE, "Consumer created", traceDetails); + trace(EventCategory.CONS, EventSubCategory.CREATED, traceDetails); } public void consumerClosed(ServerCnx cnx, Consumer consumer, Map metadata) { - if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.CONS)) return; if (!traceSystemTopics && consumer.getSubscription().getTopic().isSystemTopic()) return; - if (traceLevel == TraceLevel.OFF) return; - Map traceDetails = new TreeMap<>(); traceDetails.put("consumer", getConsumerDetails(consumer)); traceDetails.put("subscription", getSubscriptionDetails(consumer.getSubscription())); @@ -483,102 +498,16 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map traceDetails.put("messageAckRate", stats.getMessageAckRate()); traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver()); - trace(EventReasons.ADMINISTRATIVE, "Consumer closed", traceDetails); - } - - public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { - if (!jmsTracingEventList.contains(EventReasons.COMMANDS)) return; - - if (traceLevel == TraceLevel.OFF) return; - - traceCommandInternal(command, cnx); - - // todo: cache topics, and check if system topic - // without cache: - - // final String topicName = getCommandTopic(command); - // if (topicName == null) { - // // don't know if a system topic, trace it - // traceCommandInternal(command, cnx); - // return; - // } - // - // cnx.getBrokerService() - // .getTopicIfExists(topicName) - // .whenComplete( - // (topic, ex) -> { - // if (ex != null) { - // log.error("Error getting topic {} to trace command {}", topicName, command, - // ex); - // return; - // } - // - // // skip system topics if needed - // if (!traceSystemTopics && topic.isPresent() && topic.get().isSystemTopic()) - // return; - // - // traceCommandInternal(command, cnx); - // }); - } - - private static void traceCommandInternal(BaseCommand command, ServerCnx cnx) { - Map traceDetails = new TreeMap<>(); - - traceDetails.put("authMethod", cnx.getAuthMethod()); - traceDetails.put("authRole", cnx.getAuthRole()); - traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress())); - traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); - - if (command.hasType()) { - traceDetails.put("command", command.getType().name()); - traceDetails.put("parameters", getCommandDetails(command)); - } else { - traceDetails.put("type", "unknown/null"); - } - - trace(EventReasons.COMMANDS, "Pulsar command called", traceDetails); - } - - public void onConnectionClosed(ServerCnx cnx) { - if (!jmsTracingEventList.contains(EventReasons.ADMINISTRATIVE)) return; - - if (traceLevel == TraceLevel.OFF) return; - - Map traceDetails = new TreeMap<>(); - traceDetails.put("serverCnx", getConnectionDetails(cnx)); - traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); - - trace(EventReasons.ADMINISTRATIVE, "Connection closed", traceDetails); + trace(EventCategory.CONS, EventSubCategory.CLOSED, traceDetails); } /* *************************** ** Message events ******************************/ - public void beforeSendMessage( - Subscription subscription, - Entry entry, - long[] ackSet, - MessageMetadata msgMetadata, - Consumer consumer) { - if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; - - TraceLevel level = getTracingLevel(subscription); - if (level == TraceLevel.OFF) return; - - Map traceDetails = new TreeMap<>(); - - addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails); - - traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength)); - - trace(EventReasons.MESSAGE, "read", traceDetails); - } - public void onMessagePublish( Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) { - - if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return; TraceLevel level = getTracingLevel(producer); if (level == TraceLevel.OFF) return; @@ -594,7 +523,7 @@ public void onMessagePublish( "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); traceDetails.put("payload", headersAndPayloadDetails); - trace(EventReasons.MESSAGE, "received", traceDetails); + trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails); } public void messageProduced( @@ -604,7 +533,7 @@ public void messageProduced( long ledgerId, long entryId, Topic.PublishContext publishContext) { - if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return; TraceLevel level = getTracingLevel(producer); if (level == TraceLevel.OFF) return; @@ -615,12 +544,32 @@ public void messageProduced( traceDetails.put("publishContext", getPublishContextDetails(publishContext)); traceDetails.put("messageId", ledgerId + ":" + entryId); traceDetails.put("startTimeNs", startTimeNs); - trace(EventReasons.MESSAGE, "stored", traceDetails); + trace(EventCategory.MSG, EventSubCategory.STORED, traceDetails); + } + + public void beforeSendMessage( + Subscription subscription, + Entry entry, + long[] ackSet, + MessageMetadata msgMetadata, + Consumer consumer) { + if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return; + + TraceLevel level = getTracingLevel(subscription); + if (level == TraceLevel.OFF) return; + + Map traceDetails = new TreeMap<>(); + + addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails); + + traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength)); + + trace(EventCategory.MSG, EventSubCategory.READ, traceDetails); } public void messageDispatched( ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) { - if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return; TraceLevel level = getTracingLevel(consumer); if (level == TraceLevel.OFF) return; @@ -634,15 +583,17 @@ public void messageDispatched( "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); traceDetails.put("payload", headersAndPayloadDetails); - trace(EventReasons.MESSAGE, "dispatched", traceDetails); + trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails); } public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { - if (!jmsTracingEventList.contains(EventReasons.MESSAGE)) return; + if (!jmsTracingEventCategory.contains(EventCategory.MSG)) return; TraceLevel level = getTracingLevel(consumer); if (consumer != null && level == TraceLevel.OFF) return; + EventSubCategory subcategory = EventSubCategory.ACKED; + Map traceDetails = new TreeMap<>(); addMinimumConsumerSubscriptionDetails(consumer, traceDetails); @@ -650,6 +601,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { if (consumer == null) { // ack with empty consumer == message filtered by JMSFilter traceDetails.put("reason", "filtered by JMSFilter"); + subcategory = EventSubCategory.FILTERED; } else { // todo: am I right that unacked/nacked messages never go through broker interceptor? // in this case we need consumer interceptor to track nacks @@ -662,7 +614,6 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); traceDetails.put("subscriptionType", sub.getType().name()); } - Map ackDetails = new TreeMap<>(); if (ackCmd.hasAckType()) { ackDetails.put("type", ackCmd.getAckType().name()); @@ -689,7 +640,7 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { traceDetails.put("ack", ackDetails); - trace(EventReasons.MESSAGE, "acknowledged", traceDetails); + trace(EventCategory.MSG, subcategory, traceDetails); } /* *************************** @@ -697,39 +648,51 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { ******************************/ public void txnOpened(long tcId, String txnID) { - if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return; - if (traceLevel == TraceLevel.OFF) return; + if (!jmsTracingEventCategory.contains(EventCategory.TX)) return; Map traceDetails = new TreeMap<>(); - traceDetails.put("tcId", tcId); + traceDetails.put("tcId", tcId); // transaction coordinator id traceDetails.put("txnID", txnID); - trace(EventReasons.TRANSACTION, "Transaction opened", traceDetails); + trace(EventCategory.TX, EventSubCategory.OPENED, traceDetails); } public void txnEnded(String txnID, long txnAction) { - if (!jmsTracingEventList.contains(EventReasons.TRANSACTION)) return; - if (traceLevel == TraceLevel.OFF) return; + if (!jmsTracingEventCategory.contains(EventCategory.TX)) return; + final EventSubCategory subcategory; Map traceDetails = new TreeMap<>(); traceDetails.put("txnID", txnID); - traceDetails.put("txnAction", txnAction); - trace(EventReasons.TRANSACTION, "Transaction closed", traceDetails); + TxnAction action = TxnAction.valueOf((int) txnAction); + if (action == null) { + subcategory = EventSubCategory.CLOSED; + traceDetails.put("txnAction", "unknown action code " + txnAction); + } else { + traceDetails.put("txnAction", action.name()); + switch (action) { + case COMMIT: + subcategory = EventSubCategory.COMMITTED; + break; + case ABORT: + subcategory = EventSubCategory.ABORTED; + break; + default: + subcategory = EventSubCategory.CLOSED; + traceDetails.put("txnAction", "unknown action code " + txnAction + " " + action.name()); + break; + } + } + + trace(EventCategory.TX, subcategory, traceDetails); } /* *************************** ** Servlet events ******************************/ - public void onWebserviceRequest(ServletRequest request) { - // skipping, it is the same as onWebserviceResponse - // but without response status. - } - public void onWebserviceResponse(ServletRequest request, ServletResponse response) { - if (!jmsTracingEventList.contains(EventReasons.SERVLET)) return; - if (traceLevel == TraceLevel.OFF) return; + if (!jmsTracingEventCategory.contains(EventCategory.REST)) return; Map traceDetails = new TreeMap<>(); @@ -737,6 +700,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons traceDetails.put("protocol", request.getProtocol()); traceDetails.put("scheme", request.getScheme()); + // todo: log POST payload? try { HttpServletRequest req = (HttpServletRequest) FieldUtils.readField(request, "request", true); traceDetails.put("method", req.getMethod()); @@ -753,9 +717,21 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons log.error("Error getting request details", t); } - trace(EventReasons.SERVLET, "WebService response", traceDetails); + trace(EventCategory.REST, EventSubCategory.CALLED, traceDetails); + } + + /* *************************** + ** Skipped + ******************************/ + + public void onPulsarCommand(BaseCommand command, ServerCnx cnx) { + // skipping, output is not useful + } + + public void onWebserviceRequest(ServletRequest request) { + // skipping, it is the same as onWebserviceResponse + // but without response status. } - // not needed // public void onFilter(ServletRequest request, ServletResponse response, FilterChain chain) } diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 136ad5fd..f8caf8ef 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -21,18 +21,13 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; -import java.lang.reflect.Method; import java.net.InetAddress; import java.net.UnknownHostException; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.Optional; -import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -45,7 +40,6 @@ import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; -import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; @@ -56,33 +50,53 @@ @Slf4j public class TracingUtils { - public enum EventReasons { - ADMINISTRATIVE, - COMMANDS, - MESSAGE, - TRANSACTION, - SERVLET, + public enum EventCategory { + CONN, // connection creation, closure, + PROD, // producer creation, closure, + CONS, // consumer creation, closure, + TX, // (transaction creation, commit, rollback,etc), + MSG, // (message level send,dispatch,ack,expire,acktimeout, negative ack, etc), + REST, // (rest api calls), + } + + public enum EventSubCategory { + CREATED, + CLOSED, + + PRODUCED, + STORED, + + READ, + DISPATCHED, + ACKED, + FILTERED, + + OPENED, + COMMITTED, + ABORTED, + + CALLED, } @FunctionalInterface public interface Tracer { - void trace(EventReasons reason, String message); + void trace(EventCategory category, String message); } public static class Slf4jTracer implements Tracer { - private static final Map traceLoggers = new HashMap<>(); + private static final Map traceLoggers = new HashMap<>(); static { - for (EventReasons reason : EventReasons.values()) { + for (EventCategory category : EventCategory.values()) { traceLoggers.put( - reason, - org.slf4j.LoggerFactory.getLogger("jms-tracing-" + reason.name().toLowerCase())); + category, + org.slf4j.LoggerFactory.getLogger("jms-tracing-" + category.name().toLowerCase())); } } @Override - public void trace(EventReasons reason, String message) { - traceLoggers.get(reason).info(message); + public void trace(EventCategory category, String message) { + traceLoggers.get(category).info(message); } } @@ -136,397 +150,33 @@ public static String hostNameOf(String clientAddress) { } } - public static void trace(EventReasons reason, String message, Map traceDetails) { - trace(SLF4J_TRACER, reason, message, traceDetails); + public static void trace( + EventCategory category, EventSubCategory subCategory, Map traceDetails) { + trace(SLF4J_TRACER, category, subCategory, traceDetails); } public static void trace( - Tracer tracer, EventReasons reason, String message, Map traceDetails) { + Tracer tracer, + EventCategory category, + EventSubCategory subCategory, + Map traceDetails) { Map trace = new TreeMap<>(); - trace.put("eventType", message); + trace.put("event", category + "_" + subCategory); trace.put("traceDetails", traceDetails); try { String loggableJsonString = mapper.writeValueAsString(trace); - tracer.trace(reason, loggableJsonString); + tracer.trace(category, loggableJsonString); } catch (JsonProcessingException e) { log.error( - "Failed to serialize trace event type '{}' as json, traceDetails: {}", - message, + "Failed to serialize trace event '{}_{}' as json, traceDetails: {}", + category, + subCategory, traceDetails, e); } } - public static Map getCommandDetails(BaseCommand command) { - if (command == null) { - return null; - } - - Map details = new TreeMap<>(); - populateCommandDetails(command, details); - return details; - } - - private static void populateCommandDetails( - BaseCommand command, Map traceDetails) { - if (command == null) { - return; - } - - if (!command.hasType()) { - return; - } - - // trace all params otherwise - switch (command.getType()) { - case CONNECT: - populateByReflection(command.getConnect(), traceDetails); - break; - case CONNECTED: - populateByReflection(command.getConnected(), traceDetails); - break; - case SUBSCRIBE: - populateByReflection(command.getSubscribe(), traceDetails); - break; - case PRODUCER: - populateByReflection(command.getProducer(), traceDetails); - break; - case SEND: - populateByReflection(command.getSend(), traceDetails); - break; - case SEND_RECEIPT: - populateByReflection(command.getSendReceipt(), traceDetails); - break; - case SEND_ERROR: - populateByReflection(command.getSendError(), traceDetails); - break; - case MESSAGE: - populateByReflection(command.getMessage(), traceDetails); - break; - case ACK: - populateByReflection(command.getAck(), traceDetails); - break; - case FLOW: - populateByReflection(command.getFlow(), traceDetails); - break; - case UNSUBSCRIBE: - populateByReflection(command.getUnsubscribe(), traceDetails); - break; - case SUCCESS: - populateByReflection(command.getSuccess(), traceDetails); - break; - case ERROR: - populateByReflection(command.getError(), traceDetails); - break; - case CLOSE_PRODUCER: - populateByReflection(command.getCloseProducer(), traceDetails); - break; - case CLOSE_CONSUMER: - populateByReflection(command.getCloseConsumer(), traceDetails); - break; - case PRODUCER_SUCCESS: - populateByReflection(command.getProducerSuccess(), traceDetails); - break; - case PING: - populateByReflection(command.getPing(), traceDetails); - break; - case PONG: - populateByReflection(command.getPong(), traceDetails); - break; - case REDELIVER_UNACKNOWLEDGED_MESSAGES: - populateByReflection(command.getRedeliverUnacknowledgedMessages(), traceDetails); - break; - case PARTITIONED_METADATA: - populateByReflection(command.getPartitionMetadata(), traceDetails); - break; - case PARTITIONED_METADATA_RESPONSE: - populateByReflection(command.getPartitionMetadataResponse(), traceDetails); - break; - case LOOKUP: - populateByReflection(command.getLookupTopic(), traceDetails); - break; - case LOOKUP_RESPONSE: - populateByReflection(command.getLookupTopicResponse(), traceDetails); - break; - case CONSUMER_STATS: - populateByReflection(command.getConsumerStats(), traceDetails); - break; - case CONSUMER_STATS_RESPONSE: - populateByReflection(command.getConsumerStatsResponse(), traceDetails); - break; - case REACHED_END_OF_TOPIC: - populateByReflection(command.getReachedEndOfTopic(), traceDetails); - break; - case SEEK: - populateByReflection(command.getSeek(), traceDetails); - break; - case GET_LAST_MESSAGE_ID: - populateByReflection(command.getGetLastMessageId(), traceDetails); - break; - case GET_LAST_MESSAGE_ID_RESPONSE: - populateByReflection(command.getGetLastMessageIdResponse(), traceDetails); - break; - case ACTIVE_CONSUMER_CHANGE: - populateByReflection(command.getActiveConsumerChange(), traceDetails); - break; - case GET_TOPICS_OF_NAMESPACE: - populateByReflection(command.getGetTopicsOfNamespace(), traceDetails); - break; - case GET_TOPICS_OF_NAMESPACE_RESPONSE: - populateByReflection(command.getGetTopicsOfNamespaceResponse(), traceDetails); - break; - case GET_SCHEMA: - populateByReflection(command.getGetSchema(), traceDetails); - break; - case GET_SCHEMA_RESPONSE: - populateByReflection(command.getGetSchemaResponse(), traceDetails); - break; - case AUTH_CHALLENGE: - populateByReflection(command.getAuthChallenge(), traceDetails); - break; - case AUTH_RESPONSE: - populateByReflection(command.getAuthResponse(), traceDetails); - break; - case ACK_RESPONSE: - populateByReflection(command.getAckResponse(), traceDetails); - break; - case GET_OR_CREATE_SCHEMA: - populateByReflection(command.getGetOrCreateSchema(), traceDetails); - break; - case GET_OR_CREATE_SCHEMA_RESPONSE: - populateByReflection(command.getGetOrCreateSchemaResponse(), traceDetails); - break; - case NEW_TXN: - populateByReflection(command.getNewTxn(), traceDetails); - break; - case NEW_TXN_RESPONSE: - populateByReflection(command.getNewTxnResponse(), traceDetails); - break; - case ADD_PARTITION_TO_TXN: - populateByReflection(command.getAddPartitionToTxn(), traceDetails); - break; - case ADD_PARTITION_TO_TXN_RESPONSE: - populateByReflection(command.getAddPartitionToTxnResponse(), traceDetails); - break; - case ADD_SUBSCRIPTION_TO_TXN: - populateByReflection(command.getAddSubscriptionToTxn(), traceDetails); - break; - case ADD_SUBSCRIPTION_TO_TXN_RESPONSE: - populateByReflection(command.getAddSubscriptionToTxnResponse(), traceDetails); - break; - case END_TXN: - populateByReflection(command.getEndTxn(), traceDetails); - break; - case END_TXN_RESPONSE: - populateByReflection(command.getEndTxnResponse(), traceDetails); - break; - case END_TXN_ON_PARTITION: - populateByReflection(command.getEndTxnOnPartition(), traceDetails); - break; - case END_TXN_ON_PARTITION_RESPONSE: - populateByReflection(command.getEndTxnOnPartitionResponse(), traceDetails); - break; - case END_TXN_ON_SUBSCRIPTION: - populateByReflection(command.getEndTxnOnSubscription(), traceDetails); - break; - case END_TXN_ON_SUBSCRIPTION_RESPONSE: - populateByReflection(command.getEndTxnOnSubscriptionResponse(), traceDetails); - break; - case TC_CLIENT_CONNECT_REQUEST: - populateByReflection(command.getTcClientConnectRequest(), traceDetails); - break; - case TC_CLIENT_CONNECT_RESPONSE: - populateByReflection(command.getTcClientConnectResponse(), traceDetails); - break; - case WATCH_TOPIC_LIST: - populateByReflection(command.getWatchTopicList(), traceDetails); - break; - case WATCH_TOPIC_LIST_SUCCESS: - populateByReflection(command.getWatchTopicListSuccess(), traceDetails); - break; - case WATCH_TOPIC_UPDATE: - populateByReflection(command.getWatchTopicUpdate(), traceDetails); - break; - case WATCH_TOPIC_LIST_CLOSE: - populateByReflection(command.getWatchTopicListClose(), traceDetails); - break; - case TOPIC_MIGRATED: - populateByReflection(command.getTopicMigrated(), traceDetails); - break; - default: - log.error("Unknown command type: {}", command.getType()); - traceDetails.put("error", "unknownCommandType " + command.getType()); - } - } - - public static String getCommandTopic(BaseCommand command) { - if (command == null) { - return null; - } - - if (!command.hasType()) { - return null; - } - - // Currently not doing transitive topic resolution by e.g. topic pattern - // or by producerId/consumerId to the topic they are using. - // Also, _RESPONSE counterparts do not have topic, and we aren't matching to the request's by - // requestId. - switch (command.getType()) { - case SUBSCRIBE: - if (command.getSubscribe().hasTopic()) { - return command.getSubscribe().getTopic(); - } - break; - case PRODUCER: - if (command.getProducer().hasTopic()) { - return command.getProducer().getTopic(); - } - break; - case PARTITIONED_METADATA: - if (command.getPartitionMetadata().hasTopic()) { - return command.getPartitionMetadata().getTopic(); - } - break; - case LOOKUP: - if (command.getLookupTopic().hasTopic()) { - return command.getLookupTopic().getTopic(); - } - break; - case GET_SCHEMA: - if (command.getGetSchema().hasTopic()) { - return command.getGetSchema().getTopic(); - } - break; - case GET_OR_CREATE_SCHEMA: - if (command.getGetOrCreateSchema().hasTopic()) { - return command.getGetOrCreateSchema().getTopic(); - } - break; - case ADD_SUBSCRIPTION_TO_TXN: - if (command.getAddSubscriptionToTxn().getSubscriptionsCount() > 0 - && command.getAddSubscriptionToTxn().getSubscriptionsList().get(0).hasTopic()) { - Optional subscription = - command - .getAddSubscriptionToTxn() - .getSubscriptionsList() - .stream() - .filter(sub -> sub.hasTopic()) - .findFirst(); - if (subscription.isPresent()) { - return subscription.get().getTopic(); - } - } - break; - case END_TXN_ON_PARTITION: - if (command.getEndTxnOnPartition().hasTopic()) { - return command.getEndTxnOnPartition().getTopic(); - } - break; - case END_TXN_ON_SUBSCRIPTION: - if (command.getEndTxnOnSubscription().hasSubscription() - && command.getEndTxnOnSubscription().getSubscription().hasTopic()) { - return command.getEndTxnOnSubscription().getSubscription().getTopic(); - } - break; - default: - return null; - } - return null; - } - - private static final Set skipTraceFields = - Sets.newHashSet( - "authdata", - "authmethod", - "authmethodname", - "originalauthdata", - "orginalauthmethod", - "originalprincipal", - "schema"); - - private static void populateByReflection(Object command, Map traceDetails) { - if (command == null) { - return; - } - if (!command.getClass().getCanonicalName().contains("org.apache.pulsar.common.api.proto")) { - return; - } - - Method[] allMethods = command.getClass().getMethods(); - - Arrays.stream(allMethods) - .filter( - method -> { - if (!method.getName().startsWith("has")) { - return false; - } - String fieldName = method.getName().substring(3); - return !skipTraceFields.contains(fieldName.toLowerCase()); - }) - .filter( - method -> { - try { - return (boolean) method.invoke(command); - } catch (Exception e) { - return false; - } - }) - .forEach( - method -> { - String fieldName = method.getName().substring(3); - try { - Optional accessor = - Arrays.stream(allMethods) - .filter( - m -> - m.getName().equals("get" + fieldName) - || m.getName().equals("is" + fieldName)) - .findFirst(); - if (!accessor.isPresent()) { - log.warn( - "No accessor found for field (but has.. counterpart was found): {} of {}", - fieldName, - command.getClass().getCanonicalName()); - return; - } - Object value = accessor.get().invoke(command); - - if (value == null) return; - - // skip logging of binary data - if (value instanceof byte[] - || value instanceof ByteBuf - || value instanceof ByteBuffer) { - final int size; - if (value instanceof byte[]) { - size = ((byte[]) value).length; - } else if (value instanceof ByteBuf) { - size = ((ByteBuf) value).readableBytes(); - } else { - size = ((ByteBuffer) value).remaining(); - } - traceDetails.put(fieldName + "_size", size); - return; - } - - if (value - .getClass() - .getCanonicalName() - .contains("org.apache.pulsar.common.api.proto")) { - Map details = new TreeMap<>(); - populateByReflection(value, details); - traceDetails.put(fieldName, details); - } else { - traceDetails.put(fieldName, value); - } - } catch (Exception e) { - log.error("Failed to access field: {}", fieldName, e); - } - }); - } - public static Map getConnectionDetails(ServerCnx cnx) { if (cnx == null) { return null; diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java index 71e01bba..9fa3e0d4 100644 --- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -32,33 +32,33 @@ class TracingUtilsTest { private Tracer mockTracer = new Tracer() { @Override - public void trace(EventReasons reason, String msg) { + public void trace(EventCategory reason, String msg) { traces.add(msg); } }; - private static void trace(Tracer mockTracer, String msg, Map traceDetails) { - TracingUtils.trace(mockTracer, EventReasons.SERVLET, msg, traceDetails); + private static void trace(Tracer mockTracer, Map traceDetails) { + TracingUtils.trace(mockTracer, EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails); } @Test void traceTest() { traces.clear(); - trace(mockTracer, "msg", null); + trace(mockTracer, null); assertEquals(1, traces.size()); assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0)); Map map = new TreeMap<>(); traces.clear(); - trace(mockTracer, "msg", map); + trace(mockTracer, map); assertEquals(1, traces.size()); assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0)); map.put("key1", "value1"); traces.clear(); - trace(mockTracer, "msg", map); + trace(mockTracer, map); assertEquals(1, traces.size()); assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); } From a447548495ffd574e2a341e069e1f4678bce04b4 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Wed, 8 May 2024 17:52:28 -0700 Subject: [PATCH 28/29] PAYLOAD trace levele for messages, port for the host, tweaks for details --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 52 +++--- .../oss/pulsar/jms/tracing/TracingUtils.java | 153 +++++++++++------- .../pulsar/jms/tracing/TracingUtilsTest.java | 34 ++-- 3 files changed, 152 insertions(+), 87 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index dafa8ee0..cfcffa90 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -26,7 +26,7 @@ import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.getSubscriptionDetails; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.hostNameOf; import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.trace; -import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceByteBuf; +import static com.datastax.oss.pulsar.jms.tracing.TracingUtils.traceMetadataAndPayload; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -76,7 +76,7 @@ public class BrokerTracing implements BrokerInterceptor { private final Set jmsTracingEventCategory = new HashSet<>(); private TraceLevel traceLevel = defaultTraceLevel; - private int maxBinaryDataLength = 256; + private int maxPayloadLength = 256; private int cacheTraceLevelsDurationSec = 10; private boolean traceSystemTopics = false; private boolean traceSchema = false; @@ -297,9 +297,9 @@ public void initialize(PulsarService pulsarService) { loadEnabledEvents(pulsarService, jmsTracingEventCategory, traceLevel); Properties props = pulsarService.getConfiguration().getProperties(); - if (props.containsKey("jmsTracingMaxBinaryDataLength")) { - maxBinaryDataLength = Integer.parseInt(props.getProperty("jmsTracingMaxBinaryDataLength")); - log.info("Setting maxBinaryDataLength to {}", maxBinaryDataLength); + if (props.containsKey("jmsTracingMaxPayloadLength")) { + maxPayloadLength = Integer.parseInt(props.getProperty("jmsTracingMaxPayloadLength")); + log.info("Setting maxPayloadLength to {}", maxPayloadLength); } if (props.containsKey("jmsTracingTraceSystemTopics")) { traceSystemTopics = Boolean.parseBoolean(props.getProperty("jmsTracingTraceSystemTopics")); @@ -367,7 +367,9 @@ private static void addMinimumProducerDetails( if (producer.getAccessMode() != null) { traceDetails.put("accessMode", producer.getAccessMode().name()); } - traceDetails.put("clientHost", TracingUtils.hostNameOf(producer.getClientAddress())); + traceDetails.put("clientHost", + TracingUtils.hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort())); + if (producer.getTopic() != null) { traceDetails.put( "topicName", TopicName.get(producer.getTopic().getName()).getPartitionedTopicName()); @@ -387,7 +389,8 @@ private static void addMinimumConsumerSubscriptionDetails( if (consumer != null) { traceDetails.put("consumerName", consumer.consumerName()); traceDetails.put("consumerId", consumer.consumerId()); - traceDetails.put("clientHost", TracingUtils.hostNameOf(consumer.getClientAddress())); + traceDetails.put("clientHost", + TracingUtils.hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort())); traceDetails.put("authRole", consumer.cnx().getAuthRole()); } @@ -516,12 +519,17 @@ public void onMessagePublish( addMinimumProducerDetails(producer, traceDetails); - traceDetails.put("publishContext", getPublishContextDetails(publishContext)); + traceDetails.put("publishContext", getPublishContextDetails(level, publishContext)); - Map headersAndPayloadDetails = new TreeMap<>(); - traceByteBuf( - "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); - traceDetails.put("payload", headersAndPayloadDetails); + if (TraceLevel.PAYLOAD == level && headersAndPayload != null) { + Map headersAndPayloadDetails = new TreeMap<>(); + traceMetadataAndPayload( + "headersAndPayload", + headersAndPayload.slice(), + headersAndPayloadDetails, + maxPayloadLength); + traceDetails.put("payload", headersAndPayloadDetails); + } trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails); } @@ -541,9 +549,8 @@ public void messageProduced( Map traceDetails = new TreeMap<>(); addMinimumProducerDetails(producer, traceDetails); - traceDetails.put("publishContext", getPublishContextDetails(publishContext)); + traceDetails.put("publishContext", getPublishContextDetails(level, publishContext)); traceDetails.put("messageId", ledgerId + ":" + entryId); - traceDetails.put("startTimeNs", startTimeNs); trace(EventCategory.MSG, EventSubCategory.STORED, traceDetails); } @@ -562,7 +569,7 @@ public void beforeSendMessage( addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails); - traceDetails.put("entry", getEntryDetails(entry, maxBinaryDataLength)); + traceDetails.put("entry", getEntryDetails(level, entry, maxPayloadLength)); trace(EventCategory.MSG, EventSubCategory.READ, traceDetails); } @@ -578,10 +585,15 @@ public void messageDispatched( addMinimumConsumerSubscriptionDetails(consumer, traceDetails); traceDetails.put("messageId", ledgerId + ":" + entryId); - Map headersAndPayloadDetails = new TreeMap<>(); - traceByteBuf( - "headersAndPayload", headersAndPayload, headersAndPayloadDetails, maxBinaryDataLength); - traceDetails.put("payload", headersAndPayloadDetails); + if (TraceLevel.PAYLOAD == level && headersAndPayload != null) { + Map headersAndPayloadDetails = new TreeMap<>(); + traceMetadataAndPayload( + "headersAndPayload", + headersAndPayload.slice(), + headersAndPayloadDetails, + maxPayloadLength); + traceDetails.put("payload", headersAndPayloadDetails); + } trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails); } @@ -696,7 +708,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons Map traceDetails = new TreeMap<>(); - traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost())); + traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost(), request.getRemotePort())); traceDetails.put("protocol", request.getProtocol()); traceDetails.put("scheme", request.getScheme()); diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index f8caf8ef..4250e867 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -108,8 +108,9 @@ public void trace(EventCategory category, String message) { .enable(SerializationFeature.WRITE_NULL_MAP_VALUES); public enum TraceLevel { - OFF, - ON + OFF, // disabled + ON, // enabled without payload tracing + PAYLOAD, // enabled with payload tracing } private static final LoadingCache ipResolverCache = @@ -137,13 +138,28 @@ public String load(String clientAddress) { } }); - public static String hostNameOf(String clientAddress) { + public static String hostNameOf(String clientAddress, String clientSourceAddressAndPort) { + if (clientAddress == null || clientAddress.isEmpty() + || clientSourceAddressAndPort == null || !clientSourceAddressAndPort.contains(":")) { + return "unknown/null"; + } + + try { + String port = clientSourceAddressAndPort.split(":")[1]; + return ipResolverCache.get(clientAddress) + ":" + port; + } catch (Throwable t) { + log.error("Failed to resolve DNS for {}", clientAddress, t); + return clientAddress; + } + } + + public static String hostNameOf(String clientAddress, int remotePort) { if (clientAddress == null || clientAddress.isEmpty()) { return "unknown/null"; } try { - return ipResolverCache.get(clientAddress); + return ipResolverCache.get(clientAddress) + ":" + remotePort; } catch (Throwable t) { log.error("Failed to resolve DNS for {}", clientAddress, t); return clientAddress; @@ -191,9 +207,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map if (cnx == null) { return; } - traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress())); + traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort())); traceDetails.put("authRole", cnx.getAuthRole()); - traceDetails.put("principal", cnx.getPrincipal()); traceDetails.put("clientVersion", cnx.getClientVersion()); traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); traceDetails.put("authMethod", cnx.getAuthMethod()); @@ -293,7 +308,7 @@ private static void populateConsumerDetails(Consumer consumer, Map getEntryDetails(Entry entry, int maxBinaryDataLength) { + public static Map getEntryDetails( + TraceLevel level, Entry entry, int maxBinaryDataLength) { if (entry == null) { return null; } Map details = new TreeMap<>(); - populateEntryDetails(entry, details, maxBinaryDataLength); + populateEntryDetails(level, entry, details, maxBinaryDataLength); return details; } private static void populateEntryDetails( - Entry entry, Map traceDetails, int maxBinaryDataLength) { + TraceLevel level, Entry entry, Map traceDetails, int maxBinaryDataLength) { if (entry == null) { return; } @@ -409,75 +425,98 @@ private static void populateEntryDetails( traceDetails.put("length", entry.getLength()); - traceByteBuf("payload", entry.getDataBuffer(), traceDetails, maxBinaryDataLength); + if (TraceLevel.PAYLOAD == level && entry.getDataBuffer() != null) { + traceMetadataAndPayload( + "payload", entry.getDataBuffer().slice(), traceDetails, maxBinaryDataLength); + } } - public static Map getPublishContextDetails(Topic.PublishContext publishContext) { + public static Map getPublishContextDetails(TraceLevel level, Topic.PublishContext publishContext) { if (publishContext == null) { return null; } Map details = new TreeMap<>(); - populatePublishContext(publishContext, details); + populatePublishContext(level, publishContext, details); return details; } - private static void populatePublishContext( + private static void populatePublishContext(TraceLevel level, Topic.PublishContext publishContext, Map traceDetails) { - traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage()); - traceDetails.put("isChunked", publishContext.isChunked()); - traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages()); + traceDetails.put("sequenceId", publishContext.getSequenceId()); traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp()); traceDetails.put("msgSize", publishContext.getMsgSize()); - if (publishContext.getOriginalProducerName() != null) { - traceDetails.put("originalProducerName", publishContext.getOriginalProducerName()); - traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId()); + + if (TraceLevel.PAYLOAD == level) { + traceDetails.put("numberOfMessages", publishContext.getNumberOfMessages()); + traceDetails.put("isMarkerMessage", publishContext.isMarkerMessage()); + traceDetails.put("isChunked", publishContext.isChunked()); + if (publishContext.getOriginalProducerName() != null) { + traceDetails.put("originalProducerName", publishContext.getOriginalProducerName()); + traceDetails.put("originalSequenceId", publishContext.getOriginalSequenceId()); + } } - traceDetails.put("sequenceId", publishContext.getSequenceId()); } - public static void traceByteBuf( - String key, ByteBuf buf, Map traceDetails, int maxBinaryDataLength) { - if (buf == null || maxBinaryDataLength <= 0) return; + /** this will release metadataAndPayload */ + public static void traceMetadataAndPayload( + String key, + ByteBuf metadataAndPayload, + Map traceDetails, + int maxPayloadLength) { + if (metadataAndPayload == null) return; + if (maxPayloadLength <= 0) { + metadataAndPayload.release(); + return; + } try { + // advance readerIndex + MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload); + + // todo: do we need to trace this metadata? + populateMessageMetadataDetails(metadata, traceDetails); + + // Decode if needed + CompressionCodec codec = + CompressionCodecProvider.getCompressionCodec(metadata.getCompression()); + ByteBuf uncompressedPayload = + codec.decode(metadataAndPayload, metadata.getUncompressedSize()); + traceByteBuf(key, uncompressedPayload, traceDetails, maxPayloadLength); + } catch (Throwable t) { + log.error("Failed to trace metadataAndPayload", t); + } finally { + metadataAndPayload.release(); + } + } - final ByteBuf metadataAndPayload = buf.retainedDuplicate(); - ByteBuf uncompressedPayload = null; - try { - // advance readerIndex - MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload); - - // todo: do we need to trace this metadata? - populateMessageMetadataDetails(metadata, traceDetails); - - // Decode if needed - CompressionCodec codec = - CompressionCodecProvider.getCompressionCodec(metadata.getCompression()); - uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize()); - - // todo: does this require additional steps if messages are batched? - if (uncompressedPayload.readableBytes() < maxBinaryDataLength + 3) { - String dataAsString = uncompressedPayload.toString(StandardCharsets.UTF_8); - traceDetails.put(key, dataAsString); - } else { - String dataAsString = - uncompressedPayload.toString(0, maxBinaryDataLength, StandardCharsets.UTF_8); - traceDetails.put(key, dataAsString + "..."); - } - } finally { - metadataAndPayload.release(); - if (uncompressedPayload != null) { - uncompressedPayload.release(); - } + /** this will release payload */ + public static void traceByteBuf( + String key, ByteBuf payload, Map traceDetails, int maxPayloadLength) { + if (payload == null) return; + + if (maxPayloadLength <= 0) { + payload.release(); + return; + } + + try { + // todo: does this require additional steps if messages are batched? + String dataAsString = payload.toString(StandardCharsets.UTF_8); + if (dataAsString.length() > maxPayloadLength + 3) { + dataAsString = dataAsString.substring(0, maxPayloadLength) + "..."; } + traceDetails.put(key, dataAsString); } catch (Throwable t) { log.error("Failed to convert ByteBuf to string", t); - if (buf.readableBytes() < maxBinaryDataLength + 3) { - traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer())); + if (payload.readableBytes() < maxPayloadLength) { + traceDetails.put(key, "0x" + Hex.encodeHexString(payload.nioBuffer())); } else { - traceDetails.put( - key, "0x" + Hex.encodeHexString(buf.slice(0, maxBinaryDataLength).nioBuffer()) + "..."); + ByteBuf buf = payload.slice(0, maxPayloadLength / 2); + traceDetails.put(key, "0x" + Hex.encodeHexString(buf.nioBuffer()) + "..."); + buf.release(); } + } finally { + payload.release(); } } } diff --git a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java index 9fa3e0d4..3a87a5cf 100644 --- a/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java +++ b/pulsar-jms-tracing/src/test/java/com/datastax/oss/pulsar/jms/tracing/TracingUtilsTest.java @@ -20,9 +20,11 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.TreeMap; import org.junit.jupiter.api.Test; @@ -46,21 +48,22 @@ void traceTest() { traces.clear(); trace(mockTracer, null); assertEquals(1, traces.size()); - assertEquals("{\"eventType\":\"msg\",\"traceDetails\":null}", traces.get(0)); + assertEquals("{\"event\":\"MSG_PRODUCED\",\"traceDetails\":null}", traces.get(0)); Map map = new TreeMap<>(); traces.clear(); trace(mockTracer, map); assertEquals(1, traces.size()); - assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{}}", traces.get(0)); + assertEquals("{\"event\":\"MSG_PRODUCED\",\"traceDetails\":{}}", traces.get(0)); map.put("key1", "value1"); traces.clear(); trace(mockTracer, map); assertEquals(1, traces.size()); - assertEquals("{\"eventType\":\"msg\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); + assertEquals( + "{\"event\":\"MSG_PRODUCED\",\"traceDetails\":{\"key1\":\"value1\"}}", traces.get(0)); } // todo: @@ -96,29 +99,40 @@ void traceTest() { void traceByteBufTest() { Map traceDetails = new TreeMap<>(); - int maxBinaryDataLength = 1024; + int maxBinaryDataLength = 100; traceByteBuf("key", null, traceDetails, maxBinaryDataLength); assertEquals(0, traceDetails.size()); + Random rand = new Random(); + ByteBuf small = Unpooled.buffer(20); for (int i = 0; i < 20; i++) { - small.writeByte(i); + char randomChar = (char) (rand.nextInt(26) + 'a'); + small.writeByte(randomChar); } + String smallStr = small.toString(StandardCharsets.UTF_8); + assertEquals(1, small.refCnt()); + traceByteBuf("key", small, traceDetails, maxBinaryDataLength); assertEquals(1, traceDetails.size()); - assertEquals(42, ((String) traceDetails.get("key")).length()); - assertEquals("0x000102030405060708090a0b0c0d0e0f10111213", traceDetails.get("key")); + assertEquals(20, ((String) traceDetails.get("key")).length()); + assertEquals(smallStr, traceDetails.get("key")); + assertEquals(0, small.refCnt()); ByteBuf big = Unpooled.buffer(maxBinaryDataLength + 100); for (int i = 0; i < maxBinaryDataLength + 100; i++) { - big.writeByte(i); + char randomChar = (char) (rand.nextInt(26) + 'a'); + big.writeByte(randomChar); } + assertEquals(1, big.refCnt()); + String bigStr = big.toString(StandardCharsets.UTF_8); traceDetails.clear(); traceByteBuf("key", big, traceDetails, maxBinaryDataLength); assertEquals(1, traceDetails.size()); - assertEquals(2 + 2 * maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length()); - assertTrue(((String) traceDetails.get("key")).startsWith("0x000102")); + assertEquals(maxBinaryDataLength + 3, ((String) traceDetails.get("key")).length()); + assertEquals(bigStr.substring(0, maxBinaryDataLength) + "...", traceDetails.get("key")); + assertEquals(0, big.refCnt()); } } From db0ad0c103f33eeecf12deee53706edfc864bd25 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Fri, 10 May 2024 17:51:50 -0700 Subject: [PATCH 29/29] deduped messages, aded stats etc --- .../oss/pulsar/jms/tracing/BrokerTracing.java | 95 +++++++++---------- .../oss/pulsar/jms/tracing/TracingUtils.java | 33 +++---- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java index cfcffa90..13999322 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/BrokerTracing.java @@ -367,8 +367,10 @@ private static void addMinimumProducerDetails( if (producer.getAccessMode() != null) { traceDetails.put("accessMode", producer.getAccessMode().name()); } - traceDetails.put("clientHost", - TracingUtils.hostNameOf(producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort())); + traceDetails.put( + "clientHost", + TracingUtils.hostNameOf( + producer.getClientAddress(), producer.getCnx().clientSourceAddressAndPort())); if (producer.getTopic() != null) { traceDetails.put( @@ -389,8 +391,10 @@ private static void addMinimumConsumerSubscriptionDetails( if (consumer != null) { traceDetails.put("consumerName", consumer.consumerName()); traceDetails.put("consumerId", consumer.consumerId()); - traceDetails.put("clientHost", - TracingUtils.hostNameOf(consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort())); + traceDetails.put( + "clientHost", + TracingUtils.hostNameOf( + consumer.getClientAddress(), consumer.cnx().clientSourceAddressAndPort())); traceDetails.put("authRole", consumer.cnx().getAuthRole()); } @@ -451,13 +455,16 @@ public void producerClosed(ServerCnx cnx, Producer producer, Map traceDetails.put("metadata", metadata); traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); + Map statsTrace = new TreeMap<>(); PublisherStatsImpl stats = producer.getStats(); - traceDetails.put("connectedSince", stats.getConnectedSince()); - traceDetails.put("closedAt", DateFormatter.now()); - traceDetails.put("averageMsgSize", stats.getAverageMsgSize()); - traceDetails.put("msgRateIn", stats.getMsgRateIn()); - traceDetails.put("msgThroughputIn", stats.getMsgThroughputIn()); + + statsTrace.put("connectedSince", stats.getConnectedSince()); + statsTrace.put("closedAt", DateFormatter.now()); + statsTrace.put("averageMsgSize", stats.getAverageMsgSize()); + statsTrace.put("msgRateIn", stats.getMsgRateIn()); + statsTrace.put("msgThroughputIn", stats.getMsgThroughputIn()); // no message count in stats? stats.getCount() is not it + traceDetails.put("stats", statsTrace); trace(EventCategory.PROD, EventSubCategory.CLOSED, traceDetails); } @@ -490,16 +497,24 @@ public void consumerClosed(ServerCnx cnx, Consumer consumer, Map traceDetails.put("brokerUrl", cnx.getBrokerService().getPulsar().getBrokerServiceUrl()); ConsumerStatsImpl stats = consumer.getStats(); - traceDetails.put("connectedSince", stats.getConnectedSince()); - traceDetails.put("closedAt", DateFormatter.now()); - traceDetails.put("averageMsgSize", stats.getAvgMessagesPerEntry()); - traceDetails.put("msgRateOut", stats.getMsgRateOut()); - traceDetails.put("msgThroughputOut", stats.getMsgThroughputOut()); - traceDetails.put("msgOutCounter", stats.getMsgOutCounter()); - traceDetails.put("bytesOutCounter", stats.getBytesOutCounter()); - traceDetails.put("unackedMessages", stats.getUnackedMessages()); - traceDetails.put("messageAckRate", stats.getMessageAckRate()); - traceDetails.put("msgRateRedeliver", stats.getMsgRateRedeliver()); + Map statsTrace = new TreeMap<>(); + statsTrace.put("connectedSince", stats.getConnectedSince()); + statsTrace.put("closedAt", DateFormatter.now()); + statsTrace.put("averageMsgSize", stats.getAvgMessagesPerEntry()); + statsTrace.put("msgRateOut", stats.getMsgRateOut()); + statsTrace.put("msgThroughputOut", stats.getMsgThroughputOut()); + statsTrace.put("msgOutCounter", stats.getMsgOutCounter()); + statsTrace.put("bytesOutCounter", stats.getBytesOutCounter()); + statsTrace.put("unackedMessages", stats.getUnackedMessages()); + statsTrace.put("messageAckRate", stats.getMessageAckRate()); + statsTrace.put("msgRateRedeliver", stats.getMsgRateRedeliver()); + statsTrace.put("readPositionWhenJoining", stats.getReadPositionWhenJoining()); + Subscription sub = consumer.getSubscription(); + if (sub != null) { + statsTrace.put("subscriptionApproxBacklog", sub.getNumberOfEntriesInBacklog(false)); + statsTrace.put("subscriptionMsgRateExpired", sub.getExpiredMessageRate()); + } + traceDetails.put("stats", statsTrace); trace(EventCategory.CONS, EventSubCategory.CLOSED, traceDetails); } @@ -524,11 +539,11 @@ public void onMessagePublish( if (TraceLevel.PAYLOAD == level && headersAndPayload != null) { Map headersAndPayloadDetails = new TreeMap<>(); traceMetadataAndPayload( - "headersAndPayload", - headersAndPayload.slice(), + "payload", + headersAndPayload.retainedDuplicate(), headersAndPayloadDetails, maxPayloadLength); - traceDetails.put("payload", headersAndPayloadDetails); + traceDetails.put("headersAndPayload", headersAndPayloadDetails); } trace(EventCategory.MSG, EventSubCategory.PRODUCED, traceDetails); @@ -569,7 +584,9 @@ public void beforeSendMessage( addMinimumConsumerSubscriptionDetails(consumer, subscription, traceDetails); - traceDetails.put("entry", getEntryDetails(level, entry, maxPayloadLength)); + traceDetails.put("messageId", entry.getLedgerId() + ":" + entry.getEntryId()); + + traceDetails.put("headersAndPayload", getEntryDetails(level, entry, maxPayloadLength)); trace(EventCategory.MSG, EventSubCategory.READ, traceDetails); } @@ -588,11 +605,11 @@ public void messageDispatched( if (TraceLevel.PAYLOAD == level && headersAndPayload != null) { Map headersAndPayloadDetails = new TreeMap<>(); traceMetadataAndPayload( - "headersAndPayload", - headersAndPayload.slice(), + "payload", + headersAndPayload.retainedDuplicate(), headersAndPayloadDetails, maxPayloadLength); - traceDetails.put("payload", headersAndPayloadDetails); + traceDetails.put("headersAndPayload", headersAndPayloadDetails); } trace(EventCategory.MSG, EventSubCategory.DISPATCHED, traceDetails); @@ -604,35 +621,14 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { TraceLevel level = getTracingLevel(consumer); if (consumer != null && level == TraceLevel.OFF) return; - EventSubCategory subcategory = EventSubCategory.ACKED; - Map traceDetails = new TreeMap<>(); addMinimumConsumerSubscriptionDetails(consumer, traceDetails); - if (consumer == null) { - // ack with empty consumer == message filtered by JMSFilter - traceDetails.put("reason", "filtered by JMSFilter"); - subcategory = EventSubCategory.FILTERED; - } else { - // todo: am I right that unacked/nacked messages never go through broker interceptor? - // in this case we need consumer interceptor to track nacks - traceDetails.put("reason", "acked"); - } - - if (consumer != null && consumer.getSubscription() != null) { - Subscription sub = consumer.getSubscription(); - traceDetails.put("subscriptionName", sub.getName()); - traceDetails.put("topicName", TopicName.get(sub.getTopicName()).getPartitionedTopicName()); - traceDetails.put("subscriptionType", sub.getType().name()); - } Map ackDetails = new TreeMap<>(); if (ackCmd.hasAckType()) { ackDetails.put("type", ackCmd.getAckType().name()); } - if (ackCmd.hasConsumerId()) { - ackDetails.put("ackConsumerId", ackCmd.getConsumerId()); - } ackDetails.put("numAckedMessages", ackCmd.getMessageIdsCount()); ackDetails.put( "messageIds", @@ -652,6 +648,9 @@ public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { traceDetails.put("ack", ackDetails); + EventSubCategory subcategory = + consumer == null ? EventSubCategory.FILTERED : EventSubCategory.ACKED; + trace(EventCategory.MSG, subcategory, traceDetails); } @@ -708,7 +707,7 @@ public void onWebserviceResponse(ServletRequest request, ServletResponse respons Map traceDetails = new TreeMap<>(); - traceDetails.put("remoteHost", hostNameOf(request.getRemoteHost(), request.getRemotePort())); + traceDetails.put("host", hostNameOf(request.getRemoteHost(), request.getRemotePort())); traceDetails.put("protocol", request.getProtocol()); traceDetails.put("scheme", request.getScheme()); diff --git a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java index 4250e867..c6e6534b 100644 --- a/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java +++ b/pulsar-jms-tracing/src/main/java/com/datastax/oss/pulsar/jms/tracing/TracingUtils.java @@ -139,8 +139,10 @@ public String load(String clientAddress) { }); public static String hostNameOf(String clientAddress, String clientSourceAddressAndPort) { - if (clientAddress == null || clientAddress.isEmpty() - || clientSourceAddressAndPort == null || !clientSourceAddressAndPort.contains(":")) { + if (clientAddress == null + || clientAddress.isEmpty() + || clientSourceAddressAndPort == null + || !clientSourceAddressAndPort.contains(":")) { return "unknown/null"; } @@ -207,7 +209,8 @@ private static void populateConnectionDetails(ServerCnx cnx, Map if (cnx == null) { return; } - traceDetails.put("clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort())); + traceDetails.put( + "clientHost", hostNameOf(cnx.clientSourceAddress(), cnx.clientSourceAddressAndPort())); traceDetails.put("authRole", cnx.getAuthRole()); traceDetails.put("clientVersion", cnx.getClientVersion()); traceDetails.put("clientSourceAddressAndPort", cnx.clientSourceAddressAndPort()); @@ -300,15 +303,12 @@ private static void populateConsumerDetails(Consumer consumer, Map getPublishContextDetails(TraceLevel level, Topic.PublishContext publishContext) { + public static Map getPublishContextDetails( + TraceLevel level, Topic.PublishContext publishContext) { if (publishContext == null) { return null; } @@ -441,8 +442,8 @@ public static Map getPublishContextDetails(TraceLevel level, Top return details; } - private static void populatePublishContext(TraceLevel level, - Topic.PublishContext publishContext, Map traceDetails) { + private static void populatePublishContext( + TraceLevel level, Topic.PublishContext publishContext, Map traceDetails) { traceDetails.put("sequenceId", publishContext.getSequenceId()); traceDetails.put("entryTimestamp", publishContext.getEntryTimestamp()); traceDetails.put("msgSize", publishContext.getMsgSize());