Fix: Refactor HTTP client initialization

Centralize HTTP client initialization with consistent settings across various packages by introducing InitHTTPClient method. The global httpClient variable is now pre-configured once at startup, eliminating redundant client creation and closure, resulting in cleaner and more maintainable code.

Signed-off-by: Christian Roessner <c@roessner.co>

server/backend/lua.go

@@ -40,6 +40,14 @@ var LuaRequestChan chan *LuaRequest
 // LuaMainWorkerEndChan is a channel that signals the termination of the main Lua worker.
 var LuaMainWorkerEndChan chan Done
 
+// httpClient is a pre-configured instance of http.Client with custom timeout and TLS settings for making HTTP requests.
+var httpClient *http.Client
+
+// InitHTTPClient initializes the global httpClient variable with a pre-configured instance from util.NewHTTPClient.
+func InitHTTPClient() {
+	httpClient = util.NewHTTPClient()
+}
+
 // LuaRequest is a subset from the Authentication struct.
 // LuaRequest is a struct that includes various information for a request to Lua.
 type LuaRequest struct {
@@ -221,10 +229,6 @@ func handleLuaRequest(ctx context.Context, luaRequest *LuaRequest, compiledScrip
 		global.LuaBackendResultAttributes,
 	)
 
-	httpClient, closeHTTPClient := util.NewClosingHTTPClient()
-
-	defer closeHTTPClient()
-
 	registerDynamicLoader(L, ctx, luaRequest, httpClient)
 
 	setupGlobals(luaRequest, L, logs)

server/config/server.go

@@ -42,6 +42,7 @@ type ServerSection struct {
 	Frontend                  Frontend                 `mapstructure:"frontend"`
 	PrometheusTimer           PrometheusTimer          `mapstructure:"prometheus_timer"`
 	DefaultHTTPRequestHeader  DefaultHTTPRequestHeader `mapstructure:"default_http_request_header"`
+	HTTPClient                HTTPClient               `mapstructure:"http_client"`
 }
 
 // TLS represents the configuration for enabling TLS and managing certificates.
@@ -52,6 +53,14 @@ type TLS struct {
 	HTTPClientSkipVerify bool   `mapstructure:"http_client_skip_verify"`
 }
 
+type HTTPClient struct {
+	MaxConnsPerHost     int           `mapstructure:"max_connections_per_host"`
+	MaxIdleConns        int           `mapstructure:"max_idle_connections"`
+	MaxIdleConnsPerHost int           `mapstructure:"max_idle_connections_per_host"`
+	IdleConnTimeout     time.Duration `mapstructure:"idle_connection_timeout"`
+	Proxy               string        `mapstructure:"proxy"`
+}
+
 // BasicAuth represents the configuration for basic HTTP authentication.
 type BasicAuth struct {
 	Enabled  bool   `mapstructure:"enabled"`

server/core/hydra.go

@@ -47,6 +47,14 @@ import (
 	_ "golang.org/x/text/message/catalog"
 )
 
+// httpClient is a pre-configured instance of http.Client with custom timeout and TLS settings for making HTTP requests.
+var httpClient *http.Client
+
+// InitHTTPClient initializes the global httpClient variable with a pre-configured instance from util.NewHTTPClient.
+func InitHTTPClient() {
+	httpClient = util.NewHTTPClient()
+}
+
 // Scope represents a scope used in the ConsentPageData struct. It contains the name and description of the scope.
 // Scope represents the scope of an object.
 type Scope struct {
@@ -818,7 +826,7 @@ func createLanguagePassive(ctx *gin.Context, destPage string, languageTags []lan
 //
 // Note: This method assumes that the `ApiConfig` object is properly initialized with the `ctx` field set.
 func (a *ApiConfig) initialize() {
-	a.httpClient, a.closeHTTPClient = util.NewClosingHTTPClient()
+	a.httpClient = httpClient
 	a.guid = a.ctx.GetString(global.CtxGUIDKey)
 	configuration := createConfiguration(a.httpClient)
 	a.apiClient = openapi.NewAPIClient(configuration)
@@ -1747,10 +1755,6 @@ func deviceGETHandler(ctx *gin.Context) {
 		return
 	}
 
-	httpClient, closeHTTPClient := util.NewClosingHTTPClient()
-
-	defer closeHTTPClient()
-
 	configuration := createConfiguration(httpClient)
 	apiClient := openapi.NewAPIClient(configuration)
 

server/lualib/action/action.go

@@ -39,6 +39,14 @@ var (
 	RequestChan chan *Action
 )
 
+// httpClient is a pre-configured instance of http.Client with custom timeout and TLS settings for making HTTP requests.
+var httpClient *http.Client
+
+// InitHTTPClient initializes the global httpClient variable with a pre-configured instance from util.NewHTTPClient.
+func InitHTTPClient() {
+	httpClient = util.NewHTTPClient()
+}
+
 // Done is an empty struct that can be used to signal the completion of a task or operation.
 type Done struct{}
 
@@ -326,10 +334,6 @@ func (aw *Worker) handleRequest(httpRequest *http.Request) {
 
 	defer L.Close()
 
-	httpClient, closeHTTPClient := util.NewClosingHTTPClient()
-
-	defer closeHTTPClient()
-
 	aw.registerDynamicLoader(L, httpRequest, httpClient)
 
 	logs := new(lualib.CustomLogKeyValue)

server/lualib/callback/callback.go

@@ -38,6 +38,14 @@ var (
 	LuaCallback *PreCompiledLuaCallback
 )
 
+// httpClient is a pre-configured instance of http.Client with custom timeout and TLS settings for making HTTP requests.
+var httpClient *http.Client
+
+// InitHTTPClient initializes the global httpClient variable with a pre-configured instance from util.NewHTTPClient.
+func InitHTTPClient() {
+	httpClient = util.NewHTTPClient()
+}
+
 // PreCompiledLuaCallback represents a type that holds a precompiled Lua script and
 // allows safe concurrent access to the script.
 type PreCompiledLuaCallback struct {
@@ -224,10 +232,6 @@ func RunCallbackLuaRequest(ctx *gin.Context) (err error) {
 
 	defer L.Close()
 
-	httpClient, closeHHTPClient := util.NewClosingHTTPClient()
-
-	defer closeHHTPClient()
-
 	registerDynamicLoader(L, ctx, httpClient)
 
 	L.SetContext(luaCtx)

server/lualib/feature/feature.go

@@ -56,6 +56,14 @@ import (
 // If a script triggers or aborts the execution of features, the execution is halted and the method returns the appropriate values.
 var LuaFeatures *PreCompiledLuaFeatures
 
+// httpClient is a pre-configured instance of http.Client with custom timeout and TLS settings for making HTTP requests.
+var httpClient *http.Client
+
+// InitHTTPClient initializes the global httpClient variable with a pre-configured instance from util.NewHTTPClient.
+func InitHTTPClient() {
+	httpClient = util.NewHTTPClient()
+}
+
 // PreCompileLuaFeatures pre-compiles Lua features.
 // It checks if the configuration for Lua features is loaded and if the LuaFeatures variable is already set.
 // If the LuaFeatures variable is not set, it creates a new instance of PreCompiledLuaFeatures.
@@ -234,10 +242,6 @@ func (r *Request) CallFeatureLua(ctx *gin.Context) (triggered bool, abortFeature
 
 	defer L.Close()
 
-	httpClient, closeHHTPClient := util.NewClosingHTTPClient()
-
-	defer closeHHTPClient()
-
 	r.registerDynamicLoader(L, ctx, httpClient)
 
 	r.setGlobals(L)

server/lualib/filter/filter.go

@@ -37,6 +37,14 @@ import (
 	lua "github.com/yuin/gopher-lua"
 )
 
+// httpClient is a pre-configured instance of http.Client with custom timeout and TLS settings for making HTTP requests.
+var httpClient *http.Client
+
+// InitHTTPClient initializes the global httpClient variable with a pre-configured instance from util.NewHTTPClient.
+func InitHTTPClient() {
+	httpClient = util.NewHTTPClient()
+}
+
 // registerDynamicLoader registers a dynamic loader function in the Lua state.
 // The dynamic loader function allows loading Lua modules on-demand based on their names.
 // It takes an *lua.LState, *gin.Context, *Request, **lualib.LuaBackendResult, and *[]string as input parameters.
@@ -615,10 +623,6 @@ func (r *Request) CallFilterLua(ctx *gin.Context) (action bool, backendResult *l
 
 	defer L.Close()
 
-	httpClient, closeHTTPClient := util.NewClosingHTTPClient()
-
-	defer closeHTTPClient()
-
 	registerDynamicLoader(L, ctx, r, &backendResult, &removeAttributes, httpClient)
 
 	lualib.RegisterBackendResultType(L, global.LuaBackendResultAttributes)

server/main.go

@@ -1076,6 +1076,16 @@ func initializeInstanceInfo() {
 	infoMetric.Set(1)
 }
 
+// initializeHTTPClients initializes the HTTP clients for core, backend, action, callback, filter, and feature packages.
+func initializeHTTPClients() {
+	core.InitHTTPClient()
+	backend.InitHTTPClient()
+	action.InitHTTPClient()
+	callback.InitHTTPClient()
+	filter.InitHTTPClient()
+	feature.InitHTTPClient()
+}
+
 // runConnectionManager initializes the ConnectionManager, registers the server address, and starts a ticker to update connection counts.
 func runConnectionManager(ctx context.Context) {
 	manager := connmgr.GetConnectionManager()
@@ -1122,6 +1132,7 @@ func main() {
 
 	actionWorkers := initializeActionWorkers()
 
+	initializeHTTPClients()
 	setupWorkers(ctx, store, actionWorkers)
 	handleSignals(ctx, cancel, store, statsTicker, &monitoringTicker, actionWorkers)
 	setupRedis()

server/util/util.go

@@ -27,6 +27,7 @@ import (
 	"hash"
 	"net"
 	"net/http"
+	"net/url"
 	"regexp"
 	"runtime"
 	"strings"
@@ -516,22 +517,34 @@ func NewDNSResolver() (resolver *net.Resolver) {
 	return
 }
 
-// NewClosingHTTPClient creates and returns a new http.Client with a timeout of 60 seconds and custom TLS configurations.
-func NewClosingHTTPClient() (*http.Client, func()) {
+// NewHTTPClient creates and returns a new http.Client with a timeout of 60 seconds and custom TLS configurations.
+func NewHTTPClient() *http.Client {
+	var proxyFunc func(*http.Request) (*url.URL, error)
+
+	if config.LoadableConfig.Server.HTTPClient.Proxy != "" {
+		proxyURL, err := url.Parse(config.LoadableConfig.Server.HTTPClient.Proxy)
+		if err != nil {
+			proxyFunc = http.ProxyFromEnvironment
+		} else {
+			proxyFunc = http.ProxyURL(proxyURL)
+		}
+	} else {
+		proxyFunc = http.ProxyFromEnvironment
+	}
+
 	httpClient := &http.Client{
 		Timeout: 60 * time.Second,
 		Transport: &http.Transport{
+			Proxy:               proxyFunc,
+			MaxConnsPerHost:     config.LoadableConfig.Server.HTTPClient.MaxConnsPerHost,
+			MaxIdleConns:        config.LoadableConfig.Server.HTTPClient.MaxIdleConns,
+			MaxIdleConnsPerHost: config.LoadableConfig.Server.HTTPClient.MaxIdleConnsPerHost,
+			IdleConnTimeout:     config.LoadableConfig.Server.HTTPClient.IdleConnTimeout,
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: config.LoadableConfig.Server.TLS.HTTPClientSkipVerify,
 			},
 		},
 	}
 
-	return httpClient, func() {
-		if httpClient != nil {
-			if transport, ok := httpClient.Transport.(*http.Transport); ok {
-				transport.CloseIdleConnections()
-			}
-		}
-	}
+	return httpClient
 }