From c912f3e778a46aabb1944b56e3929acc3ccb9de3 Mon Sep 17 00:00:00 2001 From: Azeem Muzammil Date: Tue, 2 Jul 2024 14:17:58 +0530 Subject: [PATCH] Extract log related logics from states --- .../accesslog/HttpAccessLogFormatter.java | 156 ++++++++++++++++++ .../logging/accesslog/HttpAccessLogger.java | 148 +---------------- .../accesslog/ListenerHttpAccessLogger.java | 115 +++++++++++++ .../accesslog/SenderHttpAccessLogger.java | 125 ++++++++++++++ .../listener/states/SendingEntityBody.java | 61 +------ .../states/http2/SendingEntityBody.java | 83 ++-------- .../sender/states/ReceivingEntityBody.java | 100 ++--------- .../states/http2/ReceivingEntityBody.java | 97 ++--------- 8 files changed, 445 insertions(+), 440 deletions(-) create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java create mode 100644 native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java new file mode 100644 index 0000000000..79d5d5e4dc --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogFormatter.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_DATE_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_IP; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_METHOD; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_TIME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_URI; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_RESPONSE_BODY_SIZE; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_SCHEME; +import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_STATUS; + +public class HttpAccessLogFormatter { + + private HttpAccessLogFormatter() {} + + public static String formatAccessLogMessage(HttpAccessLogMessage inboundMessage, + List outboundMessages, HttpAccessLogFormat format, + List attributes) { + + Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); + if (format == HttpAccessLogFormat.FLAT) { + String inboundFormatted = inboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" ")); + + if (!outboundMessages.isEmpty()) { + String outboundFormatted = outboundMessages.stream() + .map(outboundMsg -> mapAccessLogMessage(outboundMsg, format, attributes)) + .map(outboundMap -> outboundMap.values().stream() + .filter(Objects::nonNull) + .collect(Collectors.joining(" "))) + .collect(Collectors.joining(" ")); + + return inboundFormatted + " \"~\" " + outboundFormatted; + } else { + return inboundFormatted; + } + } else { + Gson gson = new Gson(); + JsonObject jsonObject = new JsonObject(); + + inboundMap.forEach(jsonObject::addProperty); + + if (!outboundMessages.isEmpty()) { + JsonArray upstreamArray = new JsonArray(); + for (HttpAccessLogMessage outboundMessage : outboundMessages) { + Map outboundMap = mapAccessLogMessage(outboundMessage, format, attributes); + JsonObject outboundJson = gson.toJsonTree(outboundMap).getAsJsonObject(); + upstreamArray.add(outboundJson); + } + jsonObject.add("upstream", upstreamArray); + } + return gson.toJson(jsonObject); + } + } + + private static Map mapAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, List attributes) { + List allAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, + ATTRIBUTE_REQUEST_METHOD, ATTRIBUTE_REQUEST_URI, ATTRIBUTE_SCHEME, ATTRIBUTE_STATUS, + ATTRIBUTE_REQUEST_BODY_SIZE, ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_REQUEST_TIME, + ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR); + List defaultAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, ATTRIBUTE_STATUS, + ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT); + + Map attributeValues = new LinkedHashMap<>(); + allAttributes.forEach(attr -> attributeValues.put(attr, null)); + + if (!attributes.isEmpty()) { + attributes.forEach(attr -> { + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr)); + }); + } else { + defaultAttributes.forEach(attr -> + attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr))); + } + return attributeValues; + } + + private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLogMessage, + HttpAccessLogFormat format, String attribute) { + return switch (attribute) { + case ATTRIBUTE_IP -> httpAccessLogMessage.getIp(); + case ATTRIBUTE_DATE_TIME -> + String.format("[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]", httpAccessLogMessage.getDateTime()); + case ATTRIBUTE_REQUEST_METHOD -> httpAccessLogMessage.getRequestMethod(); + case ATTRIBUTE_REQUEST_URI -> httpAccessLogMessage.getRequestUri(); + case ATTRIBUTE_SCHEME -> httpAccessLogMessage.getScheme(); + case ATTRIBUTE_REQUEST -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), + httpAccessLogMessage.getRequestUri(), httpAccessLogMessage.getScheme()); + case ATTRIBUTE_STATUS -> String.valueOf(httpAccessLogMessage.getStatus()); + case ATTRIBUTE_REQUEST_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getRequestBodySize()); + case ATTRIBUTE_RESPONSE_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getResponseBodySize()); + case ATTRIBUTE_REQUEST_TIME -> String.valueOf(httpAccessLogMessage.getRequestTime()); + case ATTRIBUTE_HTTP_REFERRER -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpReferrer())); + case ATTRIBUTE_HTTP_USER_AGENT -> String.format(format == HttpAccessLogFormat.FLAT ? + "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpUserAgent())); + case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor()); + default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, attribute); + }; + } + + private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage httpAccessLogMessage, + String attribute) { + Map customHeaders = httpAccessLogMessage.getCustomHeaders(); + if (attribute.startsWith("http_")) { + String customHeaderKey = attribute.substring(5); + for (Map.Entry entry : customHeaders.entrySet()) { + if (entry.getKey().equalsIgnoreCase(customHeaderKey)) { + return entry.getValue(); + } + } + return "-"; + } + return null; + } + + private static String getHyphenForNull(String value) { + return value == null ? "-" : value; + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java index 468739fa00..1fde2e3267 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/HttpAccessLogger.java @@ -18,151 +18,11 @@ package io.ballerina.stdlib.http.api.logging.accesslog; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import io.netty.util.internal.logging.InternalLogLevel; -import io.netty.util.internal.logging.InternalLogger; -import io.netty.util.internal.logging.InternalLoggerFactory; +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; +public interface HttpAccessLogger { -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_DATE_TIME; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_REFERRER; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_USER_AGENT; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_HTTP_X_FORWARDED_FOR; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_IP; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_BODY_SIZE; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_METHOD; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_TIME; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_REQUEST_URI; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_RESPONSE_BODY_SIZE; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_SCHEME; -import static io.ballerina.stdlib.http.api.HttpConstants.ATTRIBUTE_STATUS; -import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; + void logAccessInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage); -public class HttpAccessLogger { - private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); - - private HttpAccessLogger() {} - - public static void log(HttpAccessLogMessage inboundMessage, List outboundMessages) { - String formattedAccessLogMessage = formatAccessLogMessage(inboundMessage, outboundMessages, - HttpAccessLogConfig.getInstance().getAccessLogFormat(), - HttpAccessLogConfig.getInstance().getAccessLogAttributes()); - ACCESS_LOGGER.log(InternalLogLevel.INFO, formattedAccessLogMessage); - } - - private static String formatAccessLogMessage(HttpAccessLogMessage inboundMessage, - List outboundMessages, HttpAccessLogFormat format, - List attributes) { - - Map inboundMap = mapAccessLogMessage(inboundMessage, format, attributes); - if (format == HttpAccessLogFormat.FLAT) { - String inboundFormatted = inboundMap.values().stream() - .filter(Objects::nonNull) - .collect(Collectors.joining(" ")); - - if (!outboundMessages.isEmpty()) { - String outboundFormatted = outboundMessages.stream() - .map(outboundMsg -> mapAccessLogMessage(outboundMsg, format, attributes)) - .map(outboundMap -> outboundMap.values().stream() - .filter(Objects::nonNull) - .collect(Collectors.joining(" "))) - .collect(Collectors.joining(" ")); - - return inboundFormatted + " \"~\" " + outboundFormatted; - } else { - return inboundFormatted; - } - } else { - Gson gson = new Gson(); - JsonObject jsonObject = new JsonObject(); - - inboundMap.forEach(jsonObject::addProperty); - - if (!outboundMessages.isEmpty()) { - JsonArray upstreamArray = new JsonArray(); - for (HttpAccessLogMessage outboundMessage : outboundMessages) { - Map outboundMap = mapAccessLogMessage(outboundMessage, format, attributes); - JsonObject outboundJson = gson.toJsonTree(outboundMap).getAsJsonObject(); - upstreamArray.add(outboundJson); - } - jsonObject.add("upstream", upstreamArray); - } - return gson.toJson(jsonObject); - } - } - - private static Map mapAccessLogMessage(HttpAccessLogMessage httpAccessLogMessage, - HttpAccessLogFormat format, List attributes) { - List allAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, - ATTRIBUTE_REQUEST_METHOD, ATTRIBUTE_REQUEST_URI, ATTRIBUTE_SCHEME, ATTRIBUTE_STATUS, - ATTRIBUTE_REQUEST_BODY_SIZE, ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_REQUEST_TIME, - ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT, ATTRIBUTE_HTTP_X_FORWARDED_FOR); - List defaultAttributes = List.of(ATTRIBUTE_IP, ATTRIBUTE_DATE_TIME, ATTRIBUTE_REQUEST, ATTRIBUTE_STATUS, - ATTRIBUTE_RESPONSE_BODY_SIZE, ATTRIBUTE_HTTP_REFERRER, ATTRIBUTE_HTTP_USER_AGENT); - - Map attributeValues = new LinkedHashMap<>(); - allAttributes.forEach(attr -> attributeValues.put(attr, null)); - - if (!attributes.isEmpty()) { - attributes.forEach(attr -> { - attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr)); - }); - } else { - defaultAttributes.forEach(attr -> - attributeValues.put(attr, formatAccessLogAttribute(httpAccessLogMessage, format, attr))); - } - return attributeValues; - } - - private static String formatAccessLogAttribute(HttpAccessLogMessage httpAccessLogMessage, - HttpAccessLogFormat format, String attribute) { - return switch (attribute) { - case ATTRIBUTE_IP -> httpAccessLogMessage.getIp(); - case ATTRIBUTE_DATE_TIME -> - String.format("[%1$td/%1$tb/%1$tY:%1$tT.%1$tL %1$tz]", httpAccessLogMessage.getDateTime()); - case ATTRIBUTE_REQUEST_METHOD -> httpAccessLogMessage.getRequestMethod(); - case ATTRIBUTE_REQUEST_URI -> httpAccessLogMessage.getRequestUri(); - case ATTRIBUTE_SCHEME -> httpAccessLogMessage.getScheme(); - case ATTRIBUTE_REQUEST -> String.format(format == HttpAccessLogFormat.FLAT ? - "\"%1$s %2$s %3$s\"" : "%1$s %2$s %3$s", httpAccessLogMessage.getRequestMethod(), - httpAccessLogMessage.getRequestUri(), httpAccessLogMessage.getScheme()); - case ATTRIBUTE_STATUS -> String.valueOf(httpAccessLogMessage.getStatus()); - case ATTRIBUTE_REQUEST_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getRequestBodySize()); - case ATTRIBUTE_RESPONSE_BODY_SIZE -> String.valueOf(httpAccessLogMessage.getResponseBodySize()); - case ATTRIBUTE_REQUEST_TIME -> String.valueOf(httpAccessLogMessage.getRequestTime()); - case ATTRIBUTE_HTTP_REFERRER -> String.format(format == HttpAccessLogFormat.FLAT ? - "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpReferrer())); - case ATTRIBUTE_HTTP_USER_AGENT -> String.format(format == HttpAccessLogFormat.FLAT ? - "\"%1$s\"" : "%1$s", getHyphenForNull(httpAccessLogMessage.getHttpUserAgent())); - case ATTRIBUTE_HTTP_X_FORWARDED_FOR -> getHyphenForNull(httpAccessLogMessage.getHttpXForwardedFor()); - default -> getCustomHeaderValueForAttribute(httpAccessLogMessage, attribute); - }; - } - - private static String getCustomHeaderValueForAttribute(HttpAccessLogMessage httpAccessLogMessage, - String attribute) { - Map customHeaders = httpAccessLogMessage.getCustomHeaders(); - if (attribute.startsWith("http_")) { - String customHeaderKey = attribute.substring(5); - for (Map.Entry entry : customHeaders.entrySet()) { - if (entry.getKey().equalsIgnoreCase(customHeaderKey)) { - return entry.getValue(); - } - } - return "-"; - } - return null; - } - - private static String getHyphenForNull(String value) { - return value == null ? "-" : value; - } + void updateAccessLogInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage); } diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java new file mode 100644 index 0000000000..98594c5ed5 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/ListenerHttpAccessLogger.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.contractimpl.common.Util; +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.util.internal.logging.InternalLogLevel; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Calendar; +import java.util.List; + +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogFormatter.formatAccessLogMessage; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.transport.contract.Constants.ACCESS_LOG; +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; + +public class ListenerHttpAccessLogger implements HttpAccessLogger { + + private static final Logger LOG = LoggerFactory.getLogger(ListenerHttpAccessLogger.class); + private static final InternalLogger ACCESS_LOGGER = InternalLoggerFactory.getInstance(ACCESS_LOG); + + private final Calendar inboundRequestArrivalTime; + private Long contentLength = 0L; + private String remoteAddress; + + public ListenerHttpAccessLogger(Calendar inboundRequestArrivalTime, String remoteAddress) { + this.inboundRequestArrivalTime = inboundRequestArrivalTime; + this.remoteAddress = remoteAddress; + } + + public ListenerHttpAccessLogger(Calendar inboundRequestArrivalTime, Long contentLength, String remoteAddress) { + this.inboundRequestArrivalTime = inboundRequestArrivalTime; + this.contentLength = contentLength; + this.remoteAddress = remoteAddress; + } + + @Override + public void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg) { + HttpHeaders headers = inboundRequestMsg.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; + } + + // Populate request parameters + String userAgent = "-"; + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + userAgent = headers.get(HttpHeaderNames.USER_AGENT); + } + String referrer = "-"; + if (headers.contains(HttpHeaderNames.REFERER)) { + referrer = headers.get(HttpHeaderNames.REFERER); + } + String method = inboundRequestMsg.getHttpMethod(); + String uri = inboundRequestMsg.getRequestUrl(); + HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); + String protocol; + if (request != null) { + protocol = request.protocolVersion().toString(); + } else { + protocol = inboundRequestMsg.getHttpVersion(); + } + + // Populate response parameters + int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); + + long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); + HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, + inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); + inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); + inboundMessage.setRequestTime(requestTime); + + List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); + + String formattedAccessLogMessage = formatAccessLogMessage(inboundMessage, outboundMessages, + HttpAccessLogConfig.getInstance().getAccessLogFormat(), + HttpAccessLogConfig.getInstance().getAccessLogAttributes()); + ACCESS_LOGGER.log(InternalLogLevel.INFO, formattedAccessLogMessage); + } + + @Override + public void updateAccessLogInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage) { + LOG.warn("updateAccessLogInfo is not a dependant action of this logger"); + } + + public void updateContentLength(HttpContent httpContent) { + contentLength += httpContent.content().readableBytes(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java new file mode 100644 index 0000000000..a05915de8f --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/http/api/logging/accesslog/SenderHttpAccessLogger.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.http.api.logging.accesslog; + +import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; +import io.netty.buffer.ByteBuf; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Calendar; +import java.util.List; + +import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; +import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; +import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; +import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; +import static io.ballerina.stdlib.http.transport.contract.Constants.TO; + +public class SenderHttpAccessLogger implements HttpAccessLogger { + + private static final Logger LOG = LoggerFactory.getLogger(SenderHttpAccessLogger.class); + + private Long contentLength = 0L; + private final SocketAddress remoteAddress; + + public SenderHttpAccessLogger(SocketAddress remoteAddress) { + this.remoteAddress = remoteAddress; + } + + @Override + public void logAccessInfo(HttpCarbonMessage requestMessage, HttpCarbonMessage responseMessage) { + LOG.warn("logAccessInfo is not a dependant action of this logger"); + } + + @Override + public void updateAccessLogInfo(HttpCarbonMessage outboundRequestMsg, HttpCarbonMessage inboundResponseMsg) { + HttpAccessLogMessage outboundAccessLogMessage = + getTypedProperty(outboundRequestMsg, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); + if (outboundAccessLogMessage == null) { + return; + } + + if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { + InetAddress inetAddress = inetSocketAddress.getAddress(); + outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); + outboundAccessLogMessage.setHost(inetAddress.getHostName()); + outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); + } + if (outboundAccessLogMessage.getIp().startsWith("/")) { + outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); + } + + // Populate with header parameters + HttpHeaders headers = outboundRequestMsg.getHeaders(); + if (headers.contains(HTTP_X_FORWARDED_FOR)) { + String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); + outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); + // If multiple IPs available, the first ip is the client + int firstCommaIndex = forwardedHops.indexOf(','); + outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? + forwardedHops.substring(0, firstCommaIndex) : forwardedHops); + } + if (headers.contains(HttpHeaderNames.USER_AGENT)) { + outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); + } + if (headers.contains(HttpHeaderNames.REFERER)) { + outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); + } + HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> + outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? + headers.get(customHeader) : "-")); + + outboundAccessLogMessage.setRequestMethod(outboundRequestMsg.getHttpMethod()); + outboundAccessLogMessage.setRequestUri((String) outboundRequestMsg.getProperty(TO)); + HttpMessage inboundResponse = inboundResponseMsg.getNettyHttpResponse(); + if (inboundResponse != null) { + outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); + } else { + outboundAccessLogMessage.setScheme(inboundResponseMsg.getHttpVersion()); + } + outboundAccessLogMessage.setRequestBodySize((long) outboundRequestMsg.getContentSize()); + outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); + outboundAccessLogMessage.setResponseBodySize(contentLength); + long requestTime = Calendar.getInstance().getTimeInMillis() - + outboundAccessLogMessage.getDateTime().getTimeInMillis(); + outboundAccessLogMessage.setRequestTime(requestTime); + + HttpCarbonMessage inboundReqMsg = + getTypedProperty(outboundRequestMsg, INBOUND_MESSAGE, HttpCarbonMessage.class); + + if (inboundReqMsg != null) { + List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); + if (outboundAccessLogMessages != null) { + outboundAccessLogMessages.add(outboundAccessLogMessage); + } + } + } + + public void updateContentLength(ByteBuf content) { + contentLength += content.readableBytes(); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java index a06e5946d8..51bfda13b8 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/SendingEntityBody.java @@ -18,14 +18,12 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogger; +import io.ballerina.stdlib.http.api.logging.accesslog.ListenerHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.Constants; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; import io.ballerina.stdlib.http.transport.contractimpl.HttpOutboundRespListener; -import io.ballerina.stdlib.http.transport.contractimpl.common.Util; import io.ballerina.stdlib.http.transport.contractimpl.listener.SourceHandler; import io.ballerina.stdlib.http.transport.internal.HandlerExecutor; import io.ballerina.stdlib.http.transport.internal.HttpTransportContextHolder; @@ -37,9 +35,6 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.LastHttpContent; @@ -50,13 +45,10 @@ import java.io.IOException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; -import java.util.Calendar; import java.util.List; import java.util.Queue; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_HEAD_METHOD; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_TO_HOST_CONNECTION_CLOSED; @@ -83,8 +75,6 @@ public class SendingEntityBody implements ListenerState { private HttpCarbonMessage outboundResponseMsg; private ChannelHandlerContext sourceContext; private SourceHandler sourceHandler; - private final Calendar inboundRequestArrivalTime; - private String remoteAddress = "-"; SendingEntityBody(HttpOutboundRespListener outboundRespListener, ListenerReqRespStateManager listenerReqRespStateManager, @@ -99,8 +89,6 @@ public class SendingEntityBody implements ListenerState { this.inboundRequestMsg = outboundRespListener.getInboundRequestMsg(); this.sourceContext = outboundRespListener.getSourceContext(); this.sourceHandler = outboundRespListener.getSourceHandler(); - this.inboundRequestArrivalTime = outboundRespListener.getInboundRequestArrivalTime(); - this.remoteAddress = outboundRespListener.getRemoteAddress(); } @Override @@ -238,7 +226,10 @@ private void checkForResponseWriteStatus(HttpCarbonMessage inboundRequestMsg, outboundRespStatusFuture.notifyHttpListener(inboundRequestMsg); } if (sourceHandler.getServerChannelInitializer().isHttpAccessLogEnabled()) { - logAccessInfo(inboundRequestMsg, outboundResponseMsg); + ListenerHttpAccessLogger accessLogger = new ListenerHttpAccessLogger( + outboundRespListener.getInboundRequestArrivalTime(), contentLength, + outboundRespListener.getRemoteAddress()); + accessLogger.logAccessInfo(inboundRequestMsg, outboundResponseMsg); } resetOutboundListenerState(); }); @@ -289,48 +280,6 @@ private void triggerPipeliningLogic(HttpCarbonMessage outboundResponseMsg) { } } - private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg) { - HttpHeaders headers = inboundRequestMsg.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; - } - - // Populate request parameters - String userAgent = "-"; - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - userAgent = headers.get(HttpHeaderNames.USER_AGENT); - } - String referrer = "-"; - if (headers.contains(HttpHeaderNames.REFERER)) { - referrer = headers.get(HttpHeaderNames.REFERER); - } - String method = inboundRequestMsg.getHttpMethod(); - String uri = inboundRequestMsg.getRequestUrl(); - HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); - String protocol; - if (request != null) { - protocol = request.protocolVersion().toString(); - } else { - protocol = inboundRequestMsg.getHttpVersion(); - } - - // Populate response parameters - int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); - - long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); - HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, - inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); - inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); - inboundMessage.setRequestTime(requestTime); - - List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); - - HttpAccessLogger.log(inboundMessage, outboundMessages); - } - private void resetOutboundListenerState() { contentList.clear(); contentLength = 0; diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java index dc9d3735cd..44b9c969eb 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/listener/states/http2/SendingEntityBody.java @@ -18,8 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.listener.states.http2; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogger; +import io.ballerina.stdlib.http.api.logging.accesslog.ListenerHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contract.ServerConnectorFuture; import io.ballerina.stdlib.http.transport.contract.exceptions.ServerConnectorException; @@ -27,7 +26,6 @@ import io.ballerina.stdlib.http.transport.contractimpl.common.Util; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil; -import io.ballerina.stdlib.http.transport.contractimpl.listener.HttpServerChannelInitializer; import io.ballerina.stdlib.http.transport.contractimpl.listener.http2.Http2SourceHandler; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2DataEventListener; import io.ballerina.stdlib.http.transport.message.Http2DataFrame; @@ -39,9 +37,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.codec.http2.Http2Connection; import io.netty.handler.codec.http2.Http2ConnectionEncoder; @@ -53,11 +49,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Calendar; -import java.util.List; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_CLIENT_CLOSED_WHILE_WRITING_OUTBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.validatePromisedStreamState; @@ -73,32 +65,30 @@ public class SendingEntityBody implements ListenerState { private final Http2MessageStateContext http2MessageStateContext; private final ChannelHandlerContext ctx; - private final HttpServerChannelInitializer serverChannelInitializer; private final Http2Connection conn; private final Http2ConnectionEncoder encoder; private final HttpResponseFuture outboundRespStatusFuture; private final HttpCarbonMessage inboundRequestMsg; - private final Calendar inboundRequestArrivalTime; private final int originalStreamId; private final Http2OutboundRespListener http2OutboundRespListener; private HttpCarbonMessage outboundResponseMsg; - - private Long contentLength = 0L; - private String remoteAddress; + private ListenerHttpAccessLogger accessLogger; SendingEntityBody(Http2OutboundRespListener http2OutboundRespListener, Http2MessageStateContext http2MessageStateContext) { this.http2OutboundRespListener = http2OutboundRespListener; this.http2MessageStateContext = http2MessageStateContext; this.ctx = http2OutboundRespListener.getChannelHandlerContext(); - this.serverChannelInitializer = http2OutboundRespListener.getServerChannelInitializer(); this.conn = http2OutboundRespListener.getConnection(); this.encoder = http2OutboundRespListener.getEncoder(); this.inboundRequestMsg = http2OutboundRespListener.getInboundRequestMsg(); this.outboundRespStatusFuture = inboundRequestMsg.getHttpOutboundRespStatusFuture(); - this.inboundRequestArrivalTime = http2OutboundRespListener.getInboundRequestArrivalTime(); this.originalStreamId = http2OutboundRespListener.getOriginalStreamId(); - this.remoteAddress = http2OutboundRespListener.getRemoteAddress(); + if (http2OutboundRespListener.getServerChannelInitializer().isHttpAccessLogEnabled()) { + this.accessLogger = new ListenerHttpAccessLogger( + http2OutboundRespListener.getInboundRequestArrivalTime(), + http2OutboundRespListener.getRemoteAddress()); + } } @Override @@ -178,8 +168,12 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, inboundRequestMsg); } http2OutboundRespListener.removeDefaultResponseWriter(); - if (serverChannelInitializer.isHttpAccessLogEnabled()) { - logAccessInfo(http2OutboundRespListener.getInboundRequestMsg(), outboundResponseMsg, streamId); + if (accessLogger != null) { + if (originalStreamId != streamId) { // Skip access logs for server push messages + LOG.debug("Access logging skipped for server push response"); + return; + } + accessLogger.logAccessInfo(http2OutboundRespListener.getInboundRequestMsg(), outboundResponseMsg); } http2MessageStateContext .setListenerState(new ResponseCompleted(http2OutboundRespListener, http2MessageStateContext)); @@ -189,7 +183,9 @@ private void writeContent(Http2OutboundRespListener http2OutboundRespListener, } private void writeData(HttpContent httpContent, int streamId, boolean endStream) throws Http2Exception { - contentLength += httpContent.content().readableBytes(); + if (accessLogger != null) { + accessLogger.updateContentLength(httpContent); + } validatePromisedStreamState(originalStreamId, streamId, conn, inboundRequestMsg); final ByteBuf content = httpContent.content(); for (Http2DataEventListener dataEventListener : http2OutboundRespListener.getHttp2ServerChannel() @@ -209,51 +205,4 @@ private void writeData(HttpContent httpContent, int streamId, boolean endStream) Util.addResponseWriteFailureListener(outboundRespStatusFuture, channelFuture, http2OutboundRespListener); } } - - private void logAccessInfo(HttpCarbonMessage inboundRequestMsg, HttpCarbonMessage outboundResponseMsg, - int streamId) { - if (originalStreamId != streamId) { // Skip access logs for server push messages - LOG.debug("Access logging skipped for server push response"); - return; - } - HttpHeaders headers = inboundRequestMsg.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - remoteAddress = firstCommaIndex != -1 ? forwardedHops.substring(0, firstCommaIndex) : forwardedHops; - } - - // Populate request parameters - String userAgent = "-"; - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - userAgent = headers.get(HttpHeaderNames.USER_AGENT); - } - String referrer = "-"; - if (headers.contains(HttpHeaderNames.REFERER)) { - referrer = headers.get(HttpHeaderNames.REFERER); - } - String method = inboundRequestMsg.getHttpMethod(); - String uri = inboundRequestMsg.getRequestUrl(); - HttpMessage request = inboundRequestMsg.getNettyHttpRequest(); - String protocol; - if (request != null) { - protocol = request.protocolVersion().toString(); - } else { - protocol = inboundRequestMsg.getHttpVersion(); - } - - // Populate response parameters - int statusCode = Util.getHttpResponseStatus(outboundResponseMsg).code(); - - long requestTime = Calendar.getInstance().getTimeInMillis() - inboundRequestArrivalTime.getTimeInMillis(); - HttpAccessLogMessage inboundMessage = new HttpAccessLogMessage(remoteAddress, - inboundRequestArrivalTime, method, uri, protocol, statusCode, contentLength, referrer, userAgent); - inboundMessage.setRequestBodySize((long) inboundRequestMsg.getContentSize()); - inboundMessage.setRequestTime(requestTime); - - List outboundMessages = getHttpAccessLogMessages(inboundRequestMsg); - - HttpAccessLogger.log(inboundMessage, outboundMessages); - } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java index 75143910e5..3764a2bd57 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/ReceivingEntityBody.java @@ -18,8 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.sender.states; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.api.logging.accesslog.SenderHttpAccessLogger; import io.ballerina.stdlib.http.transport.contract.HttpResponseFuture; import io.ballerina.stdlib.http.transport.contractimpl.common.states.SenderReqRespStateManager; import io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil; @@ -27,29 +26,14 @@ import io.ballerina.stdlib.http.transport.message.HttpCarbonMessage; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.LastHttpContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.Calendar; -import java.util.List; - -import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_STATE_HANDLER; import static io.ballerina.stdlib.http.transport.contract.Constants.IDLE_TIMEOUT_TRIGGERED_WHILE_READING_INBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.isKeepAlive; import static io.ballerina.stdlib.http.transport.contractimpl.common.Util.safelyRemoveHandlers; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.ILLEGAL_STATE_ERROR; @@ -64,11 +48,14 @@ public class ReceivingEntityBody implements SenderState { private final SenderReqRespStateManager senderReqRespStateManager; private final TargetHandler targetHandler; - private Long contentLength = 0L; + private SenderHttpAccessLogger accessLogger; ReceivingEntityBody(SenderReqRespStateManager senderReqRespStateManager, TargetHandler targetHandler) { this.senderReqRespStateManager = senderReqRespStateManager; this.targetHandler = targetHandler; + if (targetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { + accessLogger = new SenderHttpAccessLogger(targetHandler.getTargetChannel().getChannel().remoteAddress()); + } } @Override @@ -93,13 +80,12 @@ public void readInboundResponseEntityBody(ChannelHandlerContext ctx, HttpContent if (httpContent instanceof LastHttpContent) { StateUtil.setInboundTrailersToNewMessage(((LastHttpContent) httpContent).trailingHeaders(), inboundResponseMsg); - inboundResponseMsg.addHttpContent(httpContent); - contentLength += httpContent.content().readableBytes(); + addHttpContent(inboundResponseMsg, httpContent); inboundResponseMsg.setLastHttpContentArrived(); targetHandler.resetInboundMsg(); safelyRemoveHandlers(targetHandler.getTargetChannel().getChannel().pipeline(), IDLE_STATE_HANDLER); - if (targetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { - updateAccessLogInfo(targetHandler, inboundResponseMsg); + if (accessLogger != null) { + accessLogger.updateAccessLogInfo(targetHandler.getOutboundRequestMsg(), inboundResponseMsg); } senderReqRespStateManager.state = new EntityBodyReceived(senderReqRespStateManager); @@ -109,8 +95,7 @@ public void readInboundResponseEntityBody(ChannelHandlerContext ctx, HttpContent } targetHandler.getConnectionManager().returnChannel(targetHandler.getTargetChannel()); } else { - inboundResponseMsg.addHttpContent(httpContent); - contentLength += httpContent.content().readableBytes(); + addHttpContent(inboundResponseMsg, httpContent); } } @@ -129,69 +114,10 @@ public void handleIdleTimeoutConnectionClosure(TargetHandler targetHandler, IDLE_TIMEOUT_TRIGGERED_WHILE_READING_INBOUND_RESPONSE_BODY); } - private void updateAccessLogInfo(TargetHandler targetHandler, - HttpCarbonMessage inboundResponseMsg) { - HttpCarbonMessage httpOutboundRequest = targetHandler.getOutboundRequestMsg(); - HttpAccessLogMessage outboundAccessLogMessage = - getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); - if (outboundAccessLogMessage == null) { - return; - } - - SocketAddress remoteAddress = targetHandler.getTargetChannel().getChannel().remoteAddress(); - if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { - InetAddress inetAddress = inetSocketAddress.getAddress(); - outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); - outboundAccessLogMessage.setHost(inetAddress.getHostName()); - outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); - } - if (outboundAccessLogMessage.getIp().startsWith("/")) { - outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); - } - - // Populate with header parameters - HttpHeaders headers = httpOutboundRequest.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? - forwardedHops.substring(0, firstCommaIndex) : forwardedHops); - } - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); - } - if (headers.contains(HttpHeaderNames.REFERER)) { - outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); - } - HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> - outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? - headers.get(customHeader) : "-")); - - outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); - outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); - HttpMessage inboundResponse = inboundResponseMsg.getNettyHttpResponse(); - if (inboundResponse != null) { - outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); - } else { - outboundAccessLogMessage.setScheme(inboundResponseMsg.getHttpVersion()); - } - outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); - outboundAccessLogMessage.setStatus(inboundResponseMsg.getHttpStatusCode()); - outboundAccessLogMessage.setResponseBodySize(contentLength); - long requestTime = Calendar.getInstance().getTimeInMillis() - - outboundAccessLogMessage.getDateTime().getTimeInMillis(); - outboundAccessLogMessage.setRequestTime(requestTime); - - HttpCarbonMessage inboundReqMsg = - getTypedProperty(httpOutboundRequest, INBOUND_MESSAGE, HttpCarbonMessage.class); - - if (inboundReqMsg != null) { - List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); - if (outboundAccessLogMessages != null) { - outboundAccessLogMessages.add(outboundAccessLogMessage); - } + private void addHttpContent(HttpCarbonMessage inboundResponseMsg, HttpContent httpContent) { + inboundResponseMsg.addHttpContent(httpContent); + if (accessLogger != null) { + accessLogger.updateContentLength(httpContent.content()); } } } diff --git a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java index a92b649830..3a37c33faa 100644 --- a/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java +++ b/native/src/main/java/io/ballerina/stdlib/http/transport/contractimpl/sender/states/http2/ReceivingEntityBody.java @@ -18,8 +18,7 @@ package io.ballerina.stdlib.http.transport.contractimpl.sender.states.http2; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogConfig; -import io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogMessage; +import io.ballerina.stdlib.http.api.logging.accesslog.SenderHttpAccessLogger; import io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2MessageStateContext; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2ClientChannel; import io.ballerina.stdlib.http.transport.contractimpl.sender.http2.Http2TargetHandler; @@ -33,28 +32,13 @@ import io.netty.handler.codec.http.DefaultHttpContent; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.HttpContent; -import io.netty.handler.codec.http.HttpHeaderNames; -import io.netty.handler.codec.http.HttpHeaders; -import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http2.Http2Exception; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.Calendar; -import java.util.List; - -import static io.ballerina.stdlib.http.api.HttpConstants.INBOUND_MESSAGE; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getHttpAccessLogMessages; -import static io.ballerina.stdlib.http.api.logging.accesslog.HttpAccessLogUtil.getTypedProperty; -import static io.ballerina.stdlib.http.transport.contract.Constants.HTTP_X_FORWARDED_FOR; -import static io.ballerina.stdlib.http.transport.contract.Constants.OUTBOUND_ACCESS_LOG_MESSAGE; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_CLOSED_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_GOAWAY_WHILE_READING_INBOUND_RESPONSE_BODY; import static io.ballerina.stdlib.http.transport.contract.Constants.REMOTE_SERVER_SENT_RST_STREAM_WHILE_READING_INBOUND_RESPONSE_BODY; -import static io.ballerina.stdlib.http.transport.contract.Constants.TO; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.Http2StateUtil.releaseContent; import static io.ballerina.stdlib.http.transport.contractimpl.common.states.StateUtil.handleIncompleteInboundMessage; @@ -70,13 +54,17 @@ public class ReceivingEntityBody implements SenderState { private final Http2TargetHandler http2TargetHandler; private final Http2ClientChannel http2ClientChannel; private final Http2TargetHandler.Http2RequestWriter http2RequestWriter; - private Long contentLength = 0L; + private SenderHttpAccessLogger accessLogger; ReceivingEntityBody(Http2TargetHandler http2TargetHandler, Http2TargetHandler.Http2RequestWriter http2RequestWriter) { this.http2TargetHandler = http2TargetHandler; this.http2RequestWriter = http2RequestWriter; this.http2ClientChannel = http2TargetHandler.getHttp2ClientChannel(); + if (http2TargetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { + accessLogger = + new SenderHttpAccessLogger(http2TargetHandler.getHttp2ClientChannel().getChannel().remoteAddress()); + } } @Override @@ -153,7 +141,9 @@ private void onDataRead(Http2DataFrame http2DataFrame, OutboundMsgHolder outboun Http2MessageStateContext http2MessageStateContext) { int streamId = http2DataFrame.getStreamId(); ByteBuf data = http2DataFrame.getData(); - contentLength += data.readableBytes(); + if (accessLogger != null) { + accessLogger.updateContentLength(data); + } boolean endOfStream = http2DataFrame.isEndOfStream(); if (serverPush) { @@ -162,8 +152,8 @@ private void onDataRead(Http2DataFrame http2DataFrame, OutboundMsgHolder outboun onResponseDataRead(outboundMsgHolder, streamId, endOfStream, data); } if (endOfStream) { - if (http2TargetHandler.getHttpClientChannelInitializer().isHttpAccessLogEnabled()) { - updateAccessLogInfo(outboundMsgHolder); + if (accessLogger != null) { + accessLogger.updateAccessLogInfo(outboundMsgHolder.getRequest(), outboundMsgHolder.getResponse()); } http2MessageStateContext.setSenderState(new EntityBodyReceived(http2TargetHandler, http2RequestWriter)); } @@ -192,69 +182,4 @@ private void onResponseDataRead(OutboundMsgHolder outboundMsgHolder, int streamI responseMessage.addHttpContent(new DefaultHttpContent(data.retain())); } } - - private void updateAccessLogInfo(OutboundMsgHolder outboundMsgHolder) { - HttpCarbonMessage httpOutboundRequest = outboundMsgHolder.getRequest(); - HttpAccessLogMessage outboundAccessLogMessage = - getTypedProperty(httpOutboundRequest, OUTBOUND_ACCESS_LOG_MESSAGE, HttpAccessLogMessage.class); - if (outboundAccessLogMessage == null) { - return; - } - - SocketAddress remoteAddress = http2TargetHandler.getHttp2ClientChannel().getChannel().remoteAddress(); - if (remoteAddress instanceof InetSocketAddress inetSocketAddress) { - InetAddress inetAddress = inetSocketAddress.getAddress(); - outboundAccessLogMessage.setIp(inetAddress.getHostAddress()); - outboundAccessLogMessage.setHost(inetAddress.getHostName()); - outboundAccessLogMessage.setPort(inetSocketAddress.getPort()); - } - if (outboundAccessLogMessage.getIp().startsWith("/")) { - outboundAccessLogMessage.setIp(outboundAccessLogMessage.getIp().substring(1)); - } - - // Populate with header parameters - HttpHeaders headers = httpOutboundRequest.getHeaders(); - if (headers.contains(HTTP_X_FORWARDED_FOR)) { - String forwardedHops = headers.get(HTTP_X_FORWARDED_FOR); - outboundAccessLogMessage.setHttpXForwardedFor(forwardedHops); - // If multiple IPs available, the first ip is the client - int firstCommaIndex = forwardedHops.indexOf(','); - outboundAccessLogMessage.setIp(firstCommaIndex != -1 ? - forwardedHops.substring(0, firstCommaIndex) : forwardedHops); - } - if (headers.contains(HttpHeaderNames.USER_AGENT)) { - outboundAccessLogMessage.setHttpUserAgent(headers.get(HttpHeaderNames.USER_AGENT)); - } - if (headers.contains(HttpHeaderNames.REFERER)) { - outboundAccessLogMessage.setHttpReferrer(headers.get(HttpHeaderNames.REFERER)); - } - HttpAccessLogConfig.getInstance().getCustomHeaders().forEach(customHeader -> - outboundAccessLogMessage.putCustomHeader(customHeader, headers.contains(customHeader) ? - headers.get(customHeader) : "-")); - - outboundAccessLogMessage.setRequestMethod(httpOutboundRequest.getHttpMethod()); - outboundAccessLogMessage.setRequestUri((String) httpOutboundRequest.getProperty(TO)); - HttpMessage inboundResponse = outboundMsgHolder.getResponse().getNettyHttpResponse(); - if (inboundResponse != null) { - outboundAccessLogMessage.setScheme(inboundResponse.protocolVersion().toString()); - } else { - outboundAccessLogMessage.setScheme(outboundMsgHolder.getResponse().getHttpVersion()); - } - outboundAccessLogMessage.setRequestBodySize((long) httpOutboundRequest.getContentSize()); - outboundAccessLogMessage.setStatus(outboundMsgHolder.getResponse().getHttpStatusCode()); - outboundAccessLogMessage.setResponseBodySize(contentLength); - long requestTime = Calendar.getInstance().getTimeInMillis() - - outboundAccessLogMessage.getDateTime().getTimeInMillis(); - outboundAccessLogMessage.setRequestTime(requestTime); - - HttpCarbonMessage inboundReqMsg = - getTypedProperty(httpOutboundRequest, INBOUND_MESSAGE, HttpCarbonMessage.class); - - if (inboundReqMsg != null) { - List outboundAccessLogMessages = getHttpAccessLogMessages(inboundReqMsg); - if (outboundAccessLogMessages != null) { - outboundAccessLogMessages.add(outboundAccessLogMessage); - } - } - } }